From 4f2598fb612d7cca15255aed8bd389a78e80cc9a Mon Sep 17 00:00:00 2001 From: svntax Date: Sat, 21 Jun 2025 05:16:58 -0400 Subject: [PATCH 01/19] Add JavaFX dependencies and set up main class. --- pom.xml | 15 +++++++++++ .../java/com/example/gui/LlamaChatbox.java | 25 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/main/java/com/example/gui/LlamaChatbox.java diff --git a/pom.xml b/pom.xml index 216dda9..c41be6e 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,12 @@ tornado-runtime 1.1.1-dev + + + org.openjfx + javafx-controls + 21 + @@ -68,6 +74,15 @@ + + + org.openjfx + javafx-maven-plugin + 0.0.8 + + com.example.gui.LlamaChatbox + + diff --git a/src/main/java/com/example/gui/LlamaChatbox.java b/src/main/java/com/example/gui/LlamaChatbox.java new file mode 100644 index 0000000..fb8c3b6 --- /dev/null +++ b/src/main/java/com/example/gui/LlamaChatbox.java @@ -0,0 +1,25 @@ +package com.example.gui; + +import javafx.application.Application; +import javafx.scene.Scene; +import javafx.scene.control.Label; +import javafx.scene.layout.StackPane; +import javafx.stage.Stage; + +public class LlamaChatbox extends Application { + + @Override + public void start(Stage stage) { + String javaVersion = System.getProperty("java.version"); + String javafxVersion = System.getProperty("javafx.version"); + Label l = new Label("Hello, JavaFX " + javafxVersion + ", running on Java " + javaVersion + "."); + Scene scene = new Scene(new StackPane(l), 640, 480); + stage.setScene(scene); + stage.show(); + } + + public static void main(String[] args) { + launch(); + } + +} From e00486e8536f8ecd3c9c4844543d4355f4b898fa Mon Sep 17 00:00:00 2001 From: svntax Date: Mon, 23 Jun 2025 05:58:15 -0400 Subject: [PATCH 02/19] Set up MVCI framework for the GUI chatbox. Following the Model-View-Controller-Interactor framework, added new classes for each component. The view layout follows the POC image from issue #24. The model reflects the properties the user can change in the GUI, which are set up by the controller. The interactor will have the logic for triggering inference and updating the output displays. --- .../com/example/gui/ChatboxController.java | 27 +++ .../com/example/gui/ChatboxInteractor.java | 23 ++ .../java/com/example/gui/ChatboxModel.java | 78 +++++++ .../com/example/gui/ChatboxViewBuilder.java | 196 ++++++++++++++++++ .../java/com/example/gui/LlamaChatbox.java | 14 +- 5 files changed, 329 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/example/gui/ChatboxController.java create mode 100644 src/main/java/com/example/gui/ChatboxInteractor.java create mode 100644 src/main/java/com/example/gui/ChatboxModel.java create mode 100644 src/main/java/com/example/gui/ChatboxViewBuilder.java diff --git a/src/main/java/com/example/gui/ChatboxController.java b/src/main/java/com/example/gui/ChatboxController.java new file mode 100644 index 0000000..186cd8f --- /dev/null +++ b/src/main/java/com/example/gui/ChatboxController.java @@ -0,0 +1,27 @@ +package com.example.gui; + +import javafx.scene.layout.Region; + +public class ChatboxController { + + private final ChatboxViewBuilder viewBuilder; + private final ChatboxInteractor interactor; + + public ChatboxController() { + ChatboxModel model = new ChatboxModel(); + interactor = new ChatboxInteractor(model); + viewBuilder = new ChatboxViewBuilder(model, this::runInference); + } + + private void runInference(Runnable postRunAction) { + // TODO: Run llama tornado + System.out.println("Starting LLM inference."); + interactor.runLlamaTornado(); + postRunAction.run(); + } + + public Region getView() { + return viewBuilder.build(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java new file mode 100644 index 0000000..0aa7bae --- /dev/null +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -0,0 +1,23 @@ +package com.example.gui; + +public class ChatboxInteractor { + + private final ChatboxModel model; + + public ChatboxInteractor(ChatboxModel viewModel) { + this.model = viewModel; + } + + // TODO: Business logic for running inference and updating the output display + public void runLlamaTornado() { + String output = String.format("Engine: %s\nLlama3 Path: %s\nModel: %s\nPrompt: %s\n", + model.getSelectedEngine(), + model.getLlama3Path(), + model.getSelectedModel(), + model.getPromptText() + ); + System.out.printf(output); + model.setOutputText(output); + } + +} \ No newline at end of file diff --git a/src/main/java/com/example/gui/ChatboxModel.java b/src/main/java/com/example/gui/ChatboxModel.java new file mode 100644 index 0000000..cbf84fc --- /dev/null +++ b/src/main/java/com/example/gui/ChatboxModel.java @@ -0,0 +1,78 @@ +package com.example.gui; + +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +public class ChatboxModel { + + public enum Engine { TORNADO_VM, JVM } + + private final ObjectProperty selectedEngine = new SimpleObjectProperty<>(Engine.TORNADO_VM); + private final StringProperty llama3Path = new SimpleStringProperty(""); + private final StringProperty selectedModel = new SimpleStringProperty(""); + private final StringProperty promptText = new SimpleStringProperty(""); + private final StringProperty outputText = new SimpleStringProperty(""); + + public Engine getSelectedEngine() { + return selectedEngine.get(); + } + + public ObjectProperty selectedEngineProperty() { + return selectedEngine; + } + + public void setSelectedEngine(Engine engine){ + this.selectedEngine.set(engine); + } + + public String getLlama3Path() { + return llama3Path.get(); + } + + public StringProperty llama3PathProperty() { + return llama3Path; + } + + public void setLlama3Path(String path) { + this.llama3Path.set(path); + } + + public String getSelectedModel() { + return selectedModel.get(); + } + + public StringProperty selectedModelProperty() { + return selectedModel; + } + + public void setSelectedModel(String selectedModel) { + this.selectedModel.set(selectedModel); + } + + public String getPromptText() { + return promptText.get(); + } + + public StringProperty promptTextProperty() { + return promptText; + } + + public void setPromptText(String text) { + this.promptText.set(text); + } + + public String getOutputText() { + return outputText.get(); + } + + public StringProperty outputTextProperty() { + return outputText; + } + + public void setOutputText(String text) { + this.outputText.set(text); + } + +} \ No newline at end of file diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java new file mode 100644 index 0000000..a641f67 --- /dev/null +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -0,0 +1,196 @@ +package com.example.gui; + +import javafx.beans.property.StringProperty; +import javafx.geometry.Insets; +import javafx.geometry.Pos; +import javafx.scene.Node; +import javafx.scene.control.Button; +import javafx.scene.control.CheckBox; +import javafx.scene.control.ComboBox; +import javafx.scene.control.Label; +import javafx.scene.control.SplitPane; +import javafx.scene.control.TextArea; +import javafx.scene.control.TextField; +import javafx.scene.layout.HBox; +import javafx.scene.layout.Priority; +import javafx.scene.layout.Region; +import javafx.scene.layout.VBox; +import javafx.util.Builder; + +import java.util.function.Consumer; + +public class ChatboxViewBuilder implements Builder { + + private final ChatboxModel model; + private final Consumer actionHandler; + + public ChatboxViewBuilder(ChatboxModel model, Consumer actionHandler) { + this.model = model; + this.actionHandler = actionHandler; + } + + @Override + public Region build(){ + VBox results = new VBox(); + results.setPrefWidth(800); + results.setPrefHeight(600); + + SplitPane panels = new SplitPane(); + VBox.setVgrow(panels, Priority.ALWAYS); + results.getChildren().add(panels); + + panels.getItems().addAll( + createLeftPanel(), + createRightPanel() + ); + + return results; + } + + private Node createLeftPanel() { + VBox panel = new VBox(12); + panel.setPadding(new Insets(24)); + panel.getChildren().addAll( + createHeaderLabel("TornadoVM Chat"), + createEngineBox(), + createLlama3PathBox(), + createModelSelectBox(), + new Label("Prompt:"), + createPromptBox(), + createRunButton(), + new Label("Output:"), + createOutputArea() + ); + return panel; + } + + private Node createEngineBox() { + ComboBox engineDropdown = new ComboBox<>(); + engineDropdown.valueProperty().bindBidirectional(model.selectedEngineProperty()); + engineDropdown.getItems().addAll(ChatboxModel.Engine.values()); + engineDropdown.setMaxWidth(Double.MAX_VALUE); + HBox box = new HBox(8, new Label("Engine:"), engineDropdown); + box.setAlignment(Pos.CENTER_LEFT); + return box; + } + + private Node createLlama3PathBox() { + Button browseButton = new Button("Browse"); + // TODO: Browse directory + browseButton.setOnAction(e -> { + System.out.println("Browse pressed!"); + }); + + TextField pathField = boundTextField(model.llama3PathProperty()); + HBox box = new HBox(8, new Label("Llama3 Path:"), pathField, browseButton); + box.setAlignment(Pos.CENTER_LEFT); + HBox.setHgrow(pathField, Priority.ALWAYS); + pathField.setMaxWidth(Double.MAX_VALUE); + + return box; + } + + private Node createModelSelectBox() { + ComboBox modelDropdown = new ComboBox<>(); + modelDropdown.valueProperty().bindBidirectional(model.selectedModelProperty()); + // TODO: Update dropdown menu options when Llama3 path changes + modelDropdown.getItems().addAll("Llama-3.2-1B-Instruct-Q8_0.gguf", "Qwen3-0.6B-Q8_0.gguf"); // Hard-coded example strings for now + HBox.setHgrow(modelDropdown, Priority.ALWAYS); + modelDropdown.setMaxWidth(Double.MAX_VALUE); + + Button reloadButton = new Button("Reload"); + reloadButton.setOnAction(e -> { + // TODO: Scan Llama3 path for models + System.out.println("Reload pressed!"); + }); + + HBox box = new HBox(8, new Label("Model:"), modelDropdown, reloadButton); + box.setAlignment(Pos.CENTER_LEFT); + return box; + } + + private Node createPromptBox() { + TextField promptField = boundTextField(model.promptTextProperty()); + HBox.setHgrow(promptField, Priority.ALWAYS); + promptField.setMaxWidth(Double.MAX_VALUE); + return new HBox(8, promptField); + } + + private Node createRunButton() { + Button runButton = new Button("Run"); + runButton.setMaxWidth(Double.MAX_VALUE); + runButton.setOnAction(e -> { + // TODO: Run inference + System.out.println("Run pressed!"); + actionHandler.accept(() -> System.out.println("Finished running inference.")); + }); + return runButton; + } + + private Node createOutputArea() { + TextArea outputArea = new TextArea(); + outputArea.setEditable(false); + outputArea.setWrapText(true); + VBox.setVgrow(outputArea, Priority.ALWAYS); + outputArea.textProperty().bind(model.outputTextProperty()); + return outputArea; + } + + private Node createRightPanel() { + VBox panel = new VBox(8); + panel.setPadding(new Insets(24)); + panel.getChildren().addAll( + createMonitorOutputArea(), + createMonitorOptionsPanel() + ); + return panel; + } + + private TextArea createMonitorOutputArea() { + TextArea textArea = new TextArea(); + textArea.setEditable(false); + textArea.setWrapText(true); + VBox.setVgrow(textArea, Priority.ALWAYS); + return textArea; + } + + private Node createMonitorOptionsPanel() { + VBox box = new VBox(); + box.setPadding(new Insets(8)); + box.getChildren().addAll( + createSubHeaderLabel("System Monitor"), + createSystemMonitoringCheckBoxes() + ); + return box; + } + + private Node createSystemMonitoringCheckBoxes() { + HBox checkBoxes = new HBox(8); + checkBoxes.setAlignment(Pos.CENTER_LEFT); + checkBoxes.getChildren().addAll( + new CheckBox("htop"), + new CheckBox("nvtop"), + new CheckBox("GPU-Monitor") + ); + return checkBoxes; + } + + // Helper method for creating TextField objects with bound text property + private TextField boundTextField(StringProperty boundProperty) { + TextField textField = new TextField(); + textField.textProperty().bindBidirectional(boundProperty); + return textField; + } + + private Label createHeaderLabel(String text) { + Label label = new Label(text); + label.setStyle("-fx-font-size: 16pt; -fx-font-weight: bold;"); + return label; + } + + private Label createSubHeaderLabel(String text) { + Label label = new Label(text); + label.setStyle("-fx-font-size: 12pt; -fx-font-weight: bold;"); + return label; + } +} \ No newline at end of file diff --git a/src/main/java/com/example/gui/LlamaChatbox.java b/src/main/java/com/example/gui/LlamaChatbox.java index fb8c3b6..1fcbce6 100644 --- a/src/main/java/com/example/gui/LlamaChatbox.java +++ b/src/main/java/com/example/gui/LlamaChatbox.java @@ -2,24 +2,20 @@ import javafx.application.Application; import javafx.scene.Scene; -import javafx.scene.control.Label; -import javafx.scene.layout.StackPane; import javafx.stage.Stage; public class LlamaChatbox extends Application { @Override public void start(Stage stage) { - String javaVersion = System.getProperty("java.version"); - String javafxVersion = System.getProperty("javafx.version"); - Label l = new Label("Hello, JavaFX " + javafxVersion + ", running on Java " + javaVersion + "."); - Scene scene = new Scene(new StackPane(l), 640, 480); + ChatboxController controller = new ChatboxController(); + Scene scene = new Scene(controller.getView(), 800, 600); + stage.setTitle("TornadoVM Chat"); stage.setScene(scene); stage.show(); } public static void main(String[] args) { - launch(); + launch(args); } - -} +} \ No newline at end of file From f65a1f6e44b7348ac74ccc174a262c3830b22301 Mon Sep 17 00:00:00 2001 From: svntax Date: Tue, 24 Jun 2025 05:00:26 -0400 Subject: [PATCH 03/19] Implement run inference button. The Run button runs 'llama-tornado' as a new process and passes command-line options to it by reading from the chatbox model object. All response and error logs are displayed in the main output text area. --- .../com/example/gui/ChatboxController.java | 17 +++- .../com/example/gui/ChatboxInteractor.java | 84 +++++++++++++++++-- .../java/com/example/gui/ChatboxModel.java | 2 +- .../com/example/gui/ChatboxViewBuilder.java | 7 +- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxController.java b/src/main/java/com/example/gui/ChatboxController.java index 186cd8f..5129e93 100644 --- a/src/main/java/com/example/gui/ChatboxController.java +++ b/src/main/java/com/example/gui/ChatboxController.java @@ -1,5 +1,6 @@ package com.example.gui; +import javafx.concurrent.Task; import javafx.scene.layout.Region; public class ChatboxController { @@ -14,10 +15,18 @@ public ChatboxController() { } private void runInference(Runnable postRunAction) { - // TODO: Run llama tornado - System.out.println("Starting LLM inference."); - interactor.runLlamaTornado(); - postRunAction.run(); + Task inferenceTask = new Task<>() { + @Override + protected Void call() { + interactor.runLlamaTornado(); + return null; + } + }; + inferenceTask.setOnSucceeded(evt -> { + postRunAction.run(); + }); + Thread inferenceThread = new Thread(inferenceTask); + inferenceThread.start(); } public Region getView() { diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index 0aa7bae..af3f7fb 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -1,5 +1,12 @@ package com.example.gui; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + public class ChatboxInteractor { private final ChatboxModel model; @@ -8,16 +15,75 @@ public ChatboxInteractor(ChatboxModel viewModel) { this.model = viewModel; } - // TODO: Business logic for running inference and updating the output display + // Run the 'llama-tornado' script in the given Llama3 path, and stream its output to the GUI's output area. public void runLlamaTornado() { - String output = String.format("Engine: %s\nLlama3 Path: %s\nModel: %s\nPrompt: %s\n", - model.getSelectedEngine(), - model.getLlama3Path(), - model.getSelectedModel(), - model.getPromptText() - ); - System.out.printf(output); - model.setOutputText(output); + List commands = new ArrayList<>(); + + String llama3Path = model.getLlama3Path(); + if (llama3Path == null || llama3Path.isEmpty()) { + model.setOutputText("Please set the Llama3 path:"); + return; + } + + // Format for running 'llama-tornado' depends on the operating system. + if (System.getProperty("os.name").startsWith("Windows")) { + commands.add(String.format("%s\\external\\tornadovm\\.venv\\Scripts\\python", llama3Path)); + commands.add("llama-tornado"); + } else { + commands.add(String.format("%s/.llama-tornado", llama3Path)); + } + + ChatboxModel.Engine engine = model.getSelectedEngine(); + if (engine == ChatboxModel.Engine.TORNADO_VM) { + commands.add("--gpu"); + } + + // Assume that models are found in a /models directory. + String selectedModel = model.getSelectedModel(); + if (selectedModel == null || selectedModel.isEmpty()) { + model.setOutputText("Please select a model."); + return; + } + String modelPath = String.format("%s/models/%s", llama3Path, selectedModel); + String prompt = String.format("\"%s\"", model.getPromptText()); + + commands.addAll(Arrays.asList( + "--model", modelPath, + "--prompt", prompt + )); + + ProcessBuilder processBuilder = new ProcessBuilder(commands); + processBuilder.redirectErrorStream(true); + BufferedReader bufferedReader = null; + Process process; + try { + process = processBuilder.start(); + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + StringBuilder builder = new StringBuilder(); + + // Make sure to output the raw command. + builder.append("Running command: "); + builder.append(String.join(" ", processBuilder.command().toArray(new String[0]))); + builder.append(System.getProperty("line.separator")); + + String line; + while ((line = bufferedReader.readLine()) != null) { + builder.append(line); + builder.append(System.getProperty("line.separator")); + final String currentOutput = builder.toString(); + javafx.application.Platform.runLater(() -> model.setOutputText(currentOutput)); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } } } \ No newline at end of file diff --git a/src/main/java/com/example/gui/ChatboxModel.java b/src/main/java/com/example/gui/ChatboxModel.java index cbf84fc..790a1af 100644 --- a/src/main/java/com/example/gui/ChatboxModel.java +++ b/src/main/java/com/example/gui/ChatboxModel.java @@ -23,7 +23,7 @@ public ObjectProperty selectedEngineProperty() { return selectedEngine; } - public void setSelectedEngine(Engine engine){ + public void setSelectedEngine(Engine engine) { this.selectedEngine.set(engine); } diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index a641f67..a0152be 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -30,7 +30,7 @@ public ChatboxViewBuilder(ChatboxModel model, Consumer actionHandler) } @Override - public Region build(){ + public Region build() { VBox results = new VBox(); results.setPrefWidth(800); results.setPrefHeight(600); @@ -120,9 +120,8 @@ private Node createRunButton() { Button runButton = new Button("Run"); runButton.setMaxWidth(Double.MAX_VALUE); runButton.setOnAction(e -> { - // TODO: Run inference - System.out.println("Run pressed!"); - actionHandler.accept(() -> System.out.println("Finished running inference.")); + runButton.setDisable(true); + actionHandler.accept(() -> runButton.setDisable(false)); }); return runButton; } From 44c9750b112beb7acb7a0dd590be1a8a15ef69b0 Mon Sep 17 00:00:00 2001 From: svntax Date: Tue, 24 Jun 2025 05:17:01 -0400 Subject: [PATCH 04/19] Implement Browse button for Llama3 path. --- src/main/java/com/example/gui/ChatboxViewBuilder.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index a0152be..d6ea0e3 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -15,8 +15,10 @@ import javafx.scene.layout.Priority; import javafx.scene.layout.Region; import javafx.scene.layout.VBox; +import javafx.stage.DirectoryChooser; import javafx.util.Builder; +import java.io.File; import java.util.function.Consumer; public class ChatboxViewBuilder implements Builder { @@ -76,9 +78,13 @@ private Node createEngineBox() { private Node createLlama3PathBox() { Button browseButton = new Button("Browse"); - // TODO: Browse directory browseButton.setOnAction(e -> { - System.out.println("Browse pressed!"); + DirectoryChooser dirChooser = new DirectoryChooser(); + dirChooser.setTitle("Select GPULlama3.java Directory"); + File selectedDir = dirChooser.showDialog(browseButton.getScene().getWindow()); + if (selectedDir != null) { + model.setLlama3Path(selectedDir.getAbsolutePath()); + } }); TextField pathField = boundTextField(model.llama3PathProperty()); From a0ce1815fc644b321b274efea7dd762aae9e7498 Mon Sep 17 00:00:00 2001 From: svntax Date: Tue, 24 Jun 2025 06:02:20 -0400 Subject: [PATCH 05/19] Implement Reload button for scanning for model files. --- .../com/example/gui/ChatboxViewBuilder.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index d6ea0e3..9ae3cf7 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -99,15 +99,37 @@ private Node createLlama3PathBox() { private Node createModelSelectBox() { ComboBox modelDropdown = new ComboBox<>(); modelDropdown.valueProperty().bindBidirectional(model.selectedModelProperty()); - // TODO: Update dropdown menu options when Llama3 path changes - modelDropdown.getItems().addAll("Llama-3.2-1B-Instruct-Q8_0.gguf", "Qwen3-0.6B-Q8_0.gguf"); // Hard-coded example strings for now HBox.setHgrow(modelDropdown, Priority.ALWAYS); modelDropdown.setMaxWidth(Double.MAX_VALUE); Button reloadButton = new Button("Reload"); reloadButton.setOnAction(e -> { - // TODO: Scan Llama3 path for models - System.out.println("Reload pressed!"); + // Search the Llama3 path for a /models folder containing model files. + modelDropdown.getItems().clear(); + File llama3ModelsDir = new File(String.format("%s/models", model.getLlama3Path())); + if (llama3ModelsDir.exists() && llama3ModelsDir.isDirectory()) { + File[] files = llama3ModelsDir.listFiles((dir, name) -> name.endsWith(".gguf")); + if (files != null) { + for (File file : files) { + modelDropdown.getItems().add(file.getName()); + } + + int numModels = modelDropdown.getItems().size(); + String message = String.format("Found %d %s in %s", numModels, (numModels == 1 ? "model" : "models"), llama3ModelsDir); + String currentOutput = model.getOutputText(); + if (currentOutput.isEmpty()) { + model.setOutputText(message); + } else { + model.setOutputText(String.format("%s\n%s", model.getOutputText(), message)); + } + + if (numModels == 0) { + modelDropdown.getSelectionModel().clearSelection(); + } else { + modelDropdown.getSelectionModel().select(0); + } + } + } }); HBox box = new HBox(8, new Label("Model:"), modelDropdown, reloadButton); From f21cec1ad1910b6982162c5b662176f86a89ec5e Mon Sep 17 00:00:00 2001 From: svntax Date: Tue, 24 Jun 2025 06:42:47 -0400 Subject: [PATCH 06/19] Change output text area to autoscroll to the bottom. --- src/main/java/com/example/gui/ChatboxViewBuilder.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 9ae3cf7..3156adf 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -159,7 +159,11 @@ private Node createOutputArea() { outputArea.setEditable(false); outputArea.setWrapText(true); VBox.setVgrow(outputArea, Priority.ALWAYS); - outputArea.textProperty().bind(model.outputTextProperty()); + model.outputTextProperty().subscribe((newValue) -> { + outputArea.setText(newValue); + // Autoscroll the text area to the bottom. + outputArea.positionCaret(newValue.length()); + }); return outputArea; } From a2814617a02ff8834d52c1f7ac3b80522ecdfa50 Mon Sep 17 00:00:00 2001 From: svntax Date: Tue, 24 Jun 2025 06:50:55 -0400 Subject: [PATCH 07/19] Change buttons to be disabled while inference is running. --- src/main/java/com/example/gui/ChatboxViewBuilder.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 3156adf..5e50ab5 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -1,5 +1,7 @@ package com.example.gui; +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.StringProperty; import javafx.geometry.Insets; import javafx.geometry.Pos; @@ -24,6 +26,7 @@ public class ChatboxViewBuilder implements Builder { private final ChatboxModel model; + private final BooleanProperty inferenceRunning = new SimpleBooleanProperty(false); private final Consumer actionHandler; public ChatboxViewBuilder(ChatboxModel model, Consumer actionHandler) { @@ -78,6 +81,7 @@ private Node createEngineBox() { private Node createLlama3PathBox() { Button browseButton = new Button("Browse"); + browseButton.disableProperty().bind(inferenceRunning); browseButton.setOnAction(e -> { DirectoryChooser dirChooser = new DirectoryChooser(); dirChooser.setTitle("Select GPULlama3.java Directory"); @@ -103,6 +107,7 @@ private Node createModelSelectBox() { modelDropdown.setMaxWidth(Double.MAX_VALUE); Button reloadButton = new Button("Reload"); + reloadButton.disableProperty().bind(inferenceRunning); reloadButton.setOnAction(e -> { // Search the Llama3 path for a /models folder containing model files. modelDropdown.getItems().clear(); @@ -147,9 +152,10 @@ private Node createPromptBox() { private Node createRunButton() { Button runButton = new Button("Run"); runButton.setMaxWidth(Double.MAX_VALUE); + runButton.disableProperty().bind(inferenceRunning); runButton.setOnAction(e -> { - runButton.setDisable(true); - actionHandler.accept(() -> runButton.setDisable(false)); + inferenceRunning.set(true); + actionHandler.accept(() -> inferenceRunning.set(false)); }); return runButton; } From ad2292596c1a8124b89c951a3199896960ae3c5e Mon Sep 17 00:00:00 2001 From: svntax Date: Wed, 25 Jun 2025 04:11:20 -0400 Subject: [PATCH 08/19] Adjust buttons, labels, and container nodes for GUI chatbox. Set minimum widths for all buttons and labels. Replaced SplitPane container node with HBox to avoid having a divider in the middle. --- .../com/example/gui/ChatboxViewBuilder.java | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 5e50ab5..0df4311 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -9,8 +9,8 @@ import javafx.scene.control.Button; import javafx.scene.control.CheckBox; import javafx.scene.control.ComboBox; +import javafx.scene.control.Control; import javafx.scene.control.Label; -import javafx.scene.control.SplitPane; import javafx.scene.control.TextArea; import javafx.scene.control.TextField; import javafx.scene.layout.HBox; @@ -25,6 +25,8 @@ public class ChatboxViewBuilder implements Builder { + private static final int PANEL_WIDTH = 640; + private final ChatboxModel model; private final BooleanProperty inferenceRunning = new SimpleBooleanProperty(false); private final Consumer actionHandler; @@ -37,33 +39,28 @@ public ChatboxViewBuilder(ChatboxModel model, Consumer actionHandler) @Override public Region build() { VBox results = new VBox(); - results.setPrefWidth(800); - results.setPrefHeight(600); - SplitPane panels = new SplitPane(); + HBox panels = new HBox(createLeftPanel(), createRightPanel()); VBox.setVgrow(panels, Priority.ALWAYS); results.getChildren().add(panels); - panels.getItems().addAll( - createLeftPanel(), - createRightPanel() - ); - return results; } private Node createLeftPanel() { VBox panel = new VBox(12); + panel.setPrefWidth(PANEL_WIDTH); panel.setPadding(new Insets(24)); + HBox.setHgrow(panel, Priority.ALWAYS); panel.getChildren().addAll( createHeaderLabel("TornadoVM Chat"), createEngineBox(), createLlama3PathBox(), createModelSelectBox(), - new Label("Prompt:"), + createLabel("Prompt:"), createPromptBox(), createRunButton(), - new Label("Output:"), + createLabel("Output:"), createOutputArea() ); return panel; @@ -74,13 +71,14 @@ private Node createEngineBox() { engineDropdown.valueProperty().bindBidirectional(model.selectedEngineProperty()); engineDropdown.getItems().addAll(ChatboxModel.Engine.values()); engineDropdown.setMaxWidth(Double.MAX_VALUE); - HBox box = new HBox(8, new Label("Engine:"), engineDropdown); + HBox box = new HBox(8, createLabel("Engine:"), engineDropdown); box.setAlignment(Pos.CENTER_LEFT); return box; } private Node createLlama3PathBox() { Button browseButton = new Button("Browse"); + browseButton.setMinWidth(80); browseButton.disableProperty().bind(inferenceRunning); browseButton.setOnAction(e -> { DirectoryChooser dirChooser = new DirectoryChooser(); @@ -92,7 +90,7 @@ private Node createLlama3PathBox() { }); TextField pathField = boundTextField(model.llama3PathProperty()); - HBox box = new HBox(8, new Label("Llama3 Path:"), pathField, browseButton); + HBox box = new HBox(8, createLabel("Llama3 Path:"), pathField, browseButton); box.setAlignment(Pos.CENTER_LEFT); HBox.setHgrow(pathField, Priority.ALWAYS); pathField.setMaxWidth(Double.MAX_VALUE); @@ -107,6 +105,7 @@ private Node createModelSelectBox() { modelDropdown.setMaxWidth(Double.MAX_VALUE); Button reloadButton = new Button("Reload"); + reloadButton.setMinWidth(80); reloadButton.disableProperty().bind(inferenceRunning); reloadButton.setOnAction(e -> { // Search the Llama3 path for a /models folder containing model files. @@ -137,7 +136,7 @@ private Node createModelSelectBox() { } }); - HBox box = new HBox(8, new Label("Model:"), modelDropdown, reloadButton); + HBox box = new HBox(8, createLabel("Model:"), modelDropdown, reloadButton); box.setAlignment(Pos.CENTER_LEFT); return box; } @@ -175,7 +174,9 @@ private Node createOutputArea() { private Node createRightPanel() { VBox panel = new VBox(8); + panel.setPrefWidth(PANEL_WIDTH); panel.setPadding(new Insets(24)); + HBox.setHgrow(panel, Priority.ALWAYS); panel.getChildren().addAll( createMonitorOutputArea(), createMonitorOptionsPanel() @@ -219,14 +220,21 @@ private TextField boundTextField(StringProperty boundProperty) { return textField; } - private Label createHeaderLabel(String text) { + // Helper method to create Label objects with a minimum width + private Label createLabel(String text) { Label label = new Label(text); + label.setMinWidth(Control.USE_PREF_SIZE); + return label; + } + + private Label createHeaderLabel(String text) { + Label label = createLabel(text); label.setStyle("-fx-font-size: 16pt; -fx-font-weight: bold;"); return label; } private Label createSubHeaderLabel(String text) { - Label label = new Label(text); + Label label = createLabel(text); label.setStyle("-fx-font-size: 12pt; -fx-font-weight: bold;"); return label; } From 7c29d68960f315d8671656c630d8e6dfbef755c1 Mon Sep 17 00:00:00 2001 From: svntax Date: Wed, 25 Jun 2025 04:30:32 -0400 Subject: [PATCH 09/19] Add AtlantaFX for new theme styles. Set up AtlantaFX dependency, changed the GUI style to a dark theme (CupertinoDark), and set accents for buttons. --- pom.xml | 6 ++++++ src/main/java/com/example/gui/ChatboxViewBuilder.java | 4 ++++ src/main/java/com/example/gui/LlamaChatbox.java | 3 +++ 3 files changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index c41be6e..32d260c 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,12 @@ javafx-controls 21 + + + io.github.mkpaz + atlantafx-base + 2.0.1 + diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 0df4311..0d48068 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -1,5 +1,6 @@ package com.example.gui; +import atlantafx.base.theme.Styles; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.StringProperty; @@ -78,6 +79,7 @@ private Node createEngineBox() { private Node createLlama3PathBox() { Button browseButton = new Button("Browse"); + browseButton.getStyleClass().add(Styles.ACCENT); browseButton.setMinWidth(80); browseButton.disableProperty().bind(inferenceRunning); browseButton.setOnAction(e -> { @@ -105,6 +107,7 @@ private Node createModelSelectBox() { modelDropdown.setMaxWidth(Double.MAX_VALUE); Button reloadButton = new Button("Reload"); + reloadButton.getStyleClass().add(Styles.ACCENT); reloadButton.setMinWidth(80); reloadButton.disableProperty().bind(inferenceRunning); reloadButton.setOnAction(e -> { @@ -150,6 +153,7 @@ private Node createPromptBox() { private Node createRunButton() { Button runButton = new Button("Run"); + runButton.getStyleClass().add(Styles.ACCENT); runButton.setMaxWidth(Double.MAX_VALUE); runButton.disableProperty().bind(inferenceRunning); runButton.setOnAction(e -> { diff --git a/src/main/java/com/example/gui/LlamaChatbox.java b/src/main/java/com/example/gui/LlamaChatbox.java index 1fcbce6..e58abe3 100644 --- a/src/main/java/com/example/gui/LlamaChatbox.java +++ b/src/main/java/com/example/gui/LlamaChatbox.java @@ -1,5 +1,7 @@ package com.example.gui; +import atlantafx.base.theme.CupertinoDark; +import atlantafx.base.theme.PrimerDark; import javafx.application.Application; import javafx.scene.Scene; import javafx.stage.Stage; @@ -8,6 +10,7 @@ public class LlamaChatbox extends Application { @Override public void start(Stage stage) { + Application.setUserAgentStylesheet(new CupertinoDark().getUserAgentStylesheet()); ChatboxController controller = new ChatboxController(); Scene scene = new Scene(controller.getView(), 800, 600); stage.setTitle("TornadoVM Chat"); From c43f7a9d17dfa7815e80da642dc30c385bde38ea Mon Sep 17 00:00:00 2001 From: svntax Date: Wed, 25 Jun 2025 04:37:42 -0400 Subject: [PATCH 10/19] Decrease padding between left and right panels. --- src/main/java/com/example/gui/ChatboxViewBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 0d48068..d2d2657 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -51,7 +51,7 @@ public Region build() { private Node createLeftPanel() { VBox panel = new VBox(12); panel.setPrefWidth(PANEL_WIDTH); - panel.setPadding(new Insets(24)); + panel.setPadding(new Insets(24, 12, 24, 24)); HBox.setHgrow(panel, Priority.ALWAYS); panel.getChildren().addAll( createHeaderLabel("TornadoVM Chat"), @@ -179,7 +179,7 @@ private Node createOutputArea() { private Node createRightPanel() { VBox panel = new VBox(8); panel.setPrefWidth(PANEL_WIDTH); - panel.setPadding(new Insets(24)); + panel.setPadding(new Insets(24, 24, 24, 12)); HBox.setHgrow(panel, Priority.ALWAYS); panel.getChildren().addAll( createMonitorOutputArea(), From bed9b663671070363ab7223f008a5b6fb3d55b0a Mon Sep 17 00:00:00 2001 From: svntax Date: Wed, 25 Jun 2025 05:03:54 -0400 Subject: [PATCH 11/19] Remove unused import --- src/main/java/com/example/gui/LlamaChatbox.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/example/gui/LlamaChatbox.java b/src/main/java/com/example/gui/LlamaChatbox.java index e58abe3..7ac2cb2 100644 --- a/src/main/java/com/example/gui/LlamaChatbox.java +++ b/src/main/java/com/example/gui/LlamaChatbox.java @@ -1,7 +1,6 @@ package com.example.gui; import atlantafx.base.theme.CupertinoDark; -import atlantafx.base.theme.PrimerDark; import javafx.application.Application; import javafx.scene.Scene; import javafx.stage.Stage; From eb454a7208b2c1d84b84a6fca3374472363fea3c Mon Sep 17 00:00:00 2001 From: svntax Date: Thu, 26 Jun 2025 04:58:03 -0400 Subject: [PATCH 12/19] Remove the need for setting Llama3 path. Since we run the GUI directly in the project root, the Llama3 path browsing is no longer needed. Models are now scanned at the start too. --- .../com/example/gui/ChatboxInteractor.java | 12 ++----- .../java/com/example/gui/ChatboxModel.java | 13 ------- .../com/example/gui/ChatboxViewBuilder.java | 34 ++++--------------- 3 files changed, 9 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index af3f7fb..d7d921f 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -19,18 +19,12 @@ public ChatboxInteractor(ChatboxModel viewModel) { public void runLlamaTornado() { List commands = new ArrayList<>(); - String llama3Path = model.getLlama3Path(); - if (llama3Path == null || llama3Path.isEmpty()) { - model.setOutputText("Please set the Llama3 path:"); - return; - } - // Format for running 'llama-tornado' depends on the operating system. if (System.getProperty("os.name").startsWith("Windows")) { - commands.add(String.format("%s\\external\\tornadovm\\.venv\\Scripts\\python", llama3Path)); + commands.add("external\\tornadovm\\.venv\\Scripts\\python"); commands.add("llama-tornado"); } else { - commands.add(String.format("%s/.llama-tornado", llama3Path)); + commands.add("llama-tornado"); } ChatboxModel.Engine engine = model.getSelectedEngine(); @@ -44,7 +38,7 @@ public void runLlamaTornado() { model.setOutputText("Please select a model."); return; } - String modelPath = String.format("%s/models/%s", llama3Path, selectedModel); + String modelPath = String.format("./models/%s", selectedModel); String prompt = String.format("\"%s\"", model.getPromptText()); commands.addAll(Arrays.asList( diff --git a/src/main/java/com/example/gui/ChatboxModel.java b/src/main/java/com/example/gui/ChatboxModel.java index 790a1af..727caf7 100644 --- a/src/main/java/com/example/gui/ChatboxModel.java +++ b/src/main/java/com/example/gui/ChatboxModel.java @@ -10,7 +10,6 @@ public class ChatboxModel { public enum Engine { TORNADO_VM, JVM } private final ObjectProperty selectedEngine = new SimpleObjectProperty<>(Engine.TORNADO_VM); - private final StringProperty llama3Path = new SimpleStringProperty(""); private final StringProperty selectedModel = new SimpleStringProperty(""); private final StringProperty promptText = new SimpleStringProperty(""); private final StringProperty outputText = new SimpleStringProperty(""); @@ -27,18 +26,6 @@ public void setSelectedEngine(Engine engine) { this.selectedEngine.set(engine); } - public String getLlama3Path() { - return llama3Path.get(); - } - - public StringProperty llama3PathProperty() { - return llama3Path; - } - - public void setLlama3Path(String path) { - this.llama3Path.set(path); - } - public String getSelectedModel() { return selectedModel.get(); } diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index d2d2657..6f2b613 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -18,10 +18,10 @@ import javafx.scene.layout.Priority; import javafx.scene.layout.Region; import javafx.scene.layout.VBox; -import javafx.stage.DirectoryChooser; import javafx.util.Builder; import java.io.File; +import java.nio.file.Paths; import java.util.function.Consumer; public class ChatboxViewBuilder implements Builder { @@ -56,7 +56,6 @@ private Node createLeftPanel() { panel.getChildren().addAll( createHeaderLabel("TornadoVM Chat"), createEngineBox(), - createLlama3PathBox(), createModelSelectBox(), createLabel("Prompt:"), createPromptBox(), @@ -77,29 +76,6 @@ private Node createEngineBox() { return box; } - private Node createLlama3PathBox() { - Button browseButton = new Button("Browse"); - browseButton.getStyleClass().add(Styles.ACCENT); - browseButton.setMinWidth(80); - browseButton.disableProperty().bind(inferenceRunning); - browseButton.setOnAction(e -> { - DirectoryChooser dirChooser = new DirectoryChooser(); - dirChooser.setTitle("Select GPULlama3.java Directory"); - File selectedDir = dirChooser.showDialog(browseButton.getScene().getWindow()); - if (selectedDir != null) { - model.setLlama3Path(selectedDir.getAbsolutePath()); - } - }); - - TextField pathField = boundTextField(model.llama3PathProperty()); - HBox box = new HBox(8, createLabel("Llama3 Path:"), pathField, browseButton); - box.setAlignment(Pos.CENTER_LEFT); - HBox.setHgrow(pathField, Priority.ALWAYS); - pathField.setMaxWidth(Double.MAX_VALUE); - - return box; - } - private Node createModelSelectBox() { ComboBox modelDropdown = new ComboBox<>(); modelDropdown.valueProperty().bindBidirectional(model.selectedModelProperty()); @@ -111,9 +87,9 @@ private Node createModelSelectBox() { reloadButton.setMinWidth(80); reloadButton.disableProperty().bind(inferenceRunning); reloadButton.setOnAction(e -> { - // Search the Llama3 path for a /models folder containing model files. + // Search for a /models folder containing model files. modelDropdown.getItems().clear(); - File llama3ModelsDir = new File(String.format("%s/models", model.getLlama3Path())); + File llama3ModelsDir = new File("./models"); if (llama3ModelsDir.exists() && llama3ModelsDir.isDirectory()) { File[] files = llama3ModelsDir.listFiles((dir, name) -> name.endsWith(".gguf")); if (files != null) { @@ -122,7 +98,7 @@ private Node createModelSelectBox() { } int numModels = modelDropdown.getItems().size(); - String message = String.format("Found %d %s in %s", numModels, (numModels == 1 ? "model" : "models"), llama3ModelsDir); + String message = String.format("Found %d %s in %s", numModels, (numModels == 1 ? "model" : "models"), llama3ModelsDir.toPath().normalize().toAbsolutePath()); String currentOutput = model.getOutputText(); if (currentOutput.isEmpty()) { model.setOutputText(message); @@ -139,6 +115,8 @@ private Node createModelSelectBox() { } }); + reloadButton.fire(); // Trigger the reload once at the start. + HBox box = new HBox(8, createLabel("Model:"), modelDropdown, reloadButton); box.setAlignment(Pos.CENTER_LEFT); return box; From f1b565f5553cf9853e84394d0dc3a8f1469a1159 Mon Sep 17 00:00:00 2001 From: svntax Date: Thu, 26 Jun 2025 05:57:55 -0400 Subject: [PATCH 13/19] Modify LlamaApp and llama-tornado to support GUI mode. The `llama-tornado` script has a new `--gui` flag for launching the GUI chatbox, and as a result, the `--model` argument is no longer required if `--gui` is present. The main `LlamaApp` class and `Options` now check for the `--gui` flag to launch the JavaFX application. --- llama-tornado | 12 +++++++++++- src/main/java/com/example/LlamaApp.java | 19 ++++++++++++++----- src/main/java/com/example/Options.java | 7 +++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/llama-tornado b/llama-tornado index 112a7ca..5df1851 100755 --- a/llama-tornado +++ b/llama-tornado @@ -251,6 +251,9 @@ class LlamaRunner: print(f" {arg}") print() + if args.gui: + cmd.append("--gui") + # Execute the command try: result = subprocess.run(cmd, check=True) @@ -316,7 +319,8 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "--model", dest="model_path", - required=True, + required="--gui" not in sys.argv, + default="", help="Path to the LLM gguf file (e.g., Llama-3.2-1B-Instruct-Q8_0.gguf)", ) @@ -466,6 +470,12 @@ def create_parser() -> argparse.ArgumentParser: "--verbose", "-v", action="store_true", help="Verbose output" ) + # GUI options + gui_group = parser.add_argument_group("GUI Options") + gui_group.add_argument( + "--gui", action="store_true", help="Launch the GUI chatbox" + ) + return parser diff --git a/src/main/java/com/example/LlamaApp.java b/src/main/java/com/example/LlamaApp.java index 5ea0cb2..2e205bd 100644 --- a/src/main/java/com/example/LlamaApp.java +++ b/src/main/java/com/example/LlamaApp.java @@ -2,12 +2,14 @@ import com.example.aot.AOT; import com.example.core.model.tensor.FloatTensor; +import com.example.gui.LlamaChatbox; import com.example.inference.sampler.CategoricalSampler; import com.example.inference.sampler.Sampler; import com.example.inference.sampler.ToppSampler; import com.example.loader.weights.ModelLoader; import com.example.model.Model; import com.example.tornadovm.FloatArrayUtils; +import javafx.application.Application; import uk.ac.manchester.tornado.api.types.arrays.FloatArray; import java.io.IOException; @@ -145,13 +147,20 @@ private static Sampler createSampler(Model model, Options options) { */ public static void main(String[] args) throws IOException { Options options = Options.parseOptions(args); - Model model = loadModel(options); - Sampler sampler = createSampler(model, options); - if (options.interactive()) { - model.runInteractive(sampler, options); + if (options.guiMode()) { + // Launch the JavaFX application + Application.launch(LlamaChatbox.class, args); } else { - model.runInstructOnce(sampler, options); + // Run the CLI logic + Model model = loadModel(options); + Sampler sampler = createSampler(model, options); + + if (options.interactive()) { + model.runInteractive(sampler, options); + } else { + model.runInstructOnce(sampler, options); + } } } } diff --git a/src/main/java/com/example/Options.java b/src/main/java/com/example/Options.java index 284e754..ebf9546 100644 --- a/src/main/java/com/example/Options.java +++ b/src/main/java/com/example/Options.java @@ -5,7 +5,7 @@ import java.nio.file.Paths; public record Options(Path modelPath, String prompt, String systemPrompt, String suffix, boolean interactive, - float temperature, float topp, long seed, int maxTokens, boolean stream, boolean echo) { + float temperature, float topp, long seed, int maxTokens, boolean stream, boolean echo, boolean guiMode) { public static final int DEFAULT_MAX_TOKENS = 1024; @@ -41,6 +41,7 @@ static void printUsage(PrintStream out) { out.println(" --max-tokens, -n number of steps to run for < 0 = limited by context length, default " + DEFAULT_MAX_TOKENS); out.println(" --stream print tokens during generation; may cause encoding artifacts for non ASCII text, default true"); out.println(" --echo print ALL tokens to stderr, if true, recommended to set --stream=false, default false"); + out.println(" --gui run the GUI chatbox"); out.println(); } @@ -57,6 +58,7 @@ public static Options parseOptions(String[] args) { boolean interactive = false; boolean stream = true; boolean echo = false; + boolean guiMode = false; for (int i = 0; i < args.length; i++) { String optionName = args[i]; @@ -64,6 +66,7 @@ public static Options parseOptions(String[] args) { switch (optionName) { case "--interactive", "--chat", "-i" -> interactive = true; case "--instruct" -> interactive = false; + case "--gui" -> guiMode = true; case "--help", "-h" -> { printUsage(System.out); System.exit(0); @@ -95,6 +98,6 @@ public static Options parseOptions(String[] args) { } } } - return new Options(modelPath, prompt, systemPrompt, suffix, interactive, temperature, topp, seed, maxTokens, stream, echo); + return new Options(modelPath, prompt, systemPrompt, suffix, interactive, temperature, topp, seed, maxTokens, stream, echo, guiMode); } } From 03ee28c6a222aced9d54277e4ac2470cdaf943f9 Mon Sep 17 00:00:00 2001 From: svntax Date: Fri, 27 Jun 2025 06:45:20 -0400 Subject: [PATCH 14/19] Refactor chatbox interactor to run models directly. ChatboxInteractor now uses LlamaApp's methods to directly load and run models instead of indirectly running from the `llama-tornado` Python script. To show responses, a new PrintStream is created to capture output from Model. --- src/main/java/com/example/LlamaApp.java | 4 +- .../com/example/gui/ChatboxInteractor.java | 100 +++++++++++------- 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/example/LlamaApp.java b/src/main/java/com/example/LlamaApp.java index 2e205bd..7c614a0 100644 --- a/src/main/java/com/example/LlamaApp.java +++ b/src/main/java/com/example/LlamaApp.java @@ -120,7 +120,7 @@ static Sampler selectSampler(int vocabularySize, float temperature, float topp, * @throws IOException if the model fails to load * @throws IllegalStateException if AOT loading is enabled but the preloaded model is unavailable */ - private static Model loadModel(Options options) throws IOException { + public static Model loadModel(Options options) throws IOException { if (USE_AOT) { Model model = AOT.tryUsePreLoaded(options.modelPath(), options.maxTokens()); if (model == null) { @@ -131,7 +131,7 @@ private static Model loadModel(Options options) throws IOException { return ModelLoader.loadModel(options.modelPath(), options.maxTokens(), true); } - private static Sampler createSampler(Model model, Options options) { + public static Sampler createSampler(Model model, Options options) { return selectSampler(model.configuration().vocabularySize(), options.temperature(), options.topp(), options.seed()); } diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index d7d921f..29438fb 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -1,8 +1,12 @@ package com.example.gui; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; +import com.example.LlamaApp; +import com.example.Options; +import com.example.inference.sampler.Sampler; +import com.example.model.Model; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -15,28 +19,20 @@ public ChatboxInteractor(ChatboxModel viewModel) { this.model = viewModel; } - // Run the 'llama-tornado' script in the given Llama3 path, and stream its output to the GUI's output area. - public void runLlamaTornado() { + private String[] buildCommands() { List commands = new ArrayList<>(); - // Format for running 'llama-tornado' depends on the operating system. - if (System.getProperty("os.name").startsWith("Windows")) { - commands.add("external\\tornadovm\\.venv\\Scripts\\python"); - commands.add("llama-tornado"); - } else { - commands.add("llama-tornado"); - } - ChatboxModel.Engine engine = model.getSelectedEngine(); if (engine == ChatboxModel.Engine.TORNADO_VM) { - commands.add("--gpu"); + // TODO: LlamaApp.USE_TORNADOVM is a final constant, but the GUI needs to be able to set this value + //commands.add("--gpu"); } // Assume that models are found in a /models directory. String selectedModel = model.getSelectedModel(); if (selectedModel == null || selectedModel.isEmpty()) { model.setOutputText("Please select a model."); - return; + return null; } String modelPath = String.format("./models/%s", selectedModel); String prompt = String.format("\"%s\"", model.getPromptText()); @@ -46,37 +42,65 @@ public void runLlamaTornado() { "--prompt", prompt )); - ProcessBuilder processBuilder = new ProcessBuilder(commands); - processBuilder.redirectErrorStream(true); - BufferedReader bufferedReader = null; - Process process; + return commands.toArray(new String[commands.size()]); + } + + // Load and run a model while capturing its output text to a custom stream. + public void runLlamaTornado() { + // Save the original System.out stream + PrintStream originalOut = System.out; try { - process = processBuilder.start(); - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String[] commands = buildCommands(); + if (commands == null) { + // commands is null if no model was found, so exit this process + return; + } + StringBuilder builder = new StringBuilder(); - // Make sure to output the raw command. - builder.append("Running command: "); - builder.append(String.join(" ", processBuilder.command().toArray(new String[0]))); + builder.append("Processing... please wait"); builder.append(System.getProperty("line.separator")); - String line; - while ((line = bufferedReader.readLine()) != null) { - builder.append(line); - builder.append(System.getProperty("line.separator")); - final String currentOutput = builder.toString(); - javafx.application.Platform.runLater(() -> model.setOutputText(currentOutput)); + // Create a custom PrintStream to capture output from loading the model and running it. + PrintStream customStream = new PrintStream(new ByteArrayOutputStream()) { + @Override + public void println(String str) { + process(str + "\n"); + } + + @Override + public void print(String str) { + process(str); + } + + private void process(String str) { + // Capture the output stream into the GUI output area. + builder.append(str); + final String currentOutput = builder.toString(); + javafx.application.Platform.runLater(() -> model.setOutputText(currentOutput)); + } + }; + + // Redirect System.out to the custom print stream. + System.setOut(customStream); + System.setErr(customStream); + + Options options = Options.parseOptions(commands); + + // Load the model and run. + Model llm = LlamaApp.loadModel(options); + Sampler sampler = LlamaApp.createSampler(llm, options); + if (options.interactive()) { + llm.runInteractive(sampler, options); + } else { + llm.runInstructOnce(sampler, options); } - } catch (IOException e) { + } catch (Exception e) { + // Catch all exceptions so that they're logged in the output area. e.printStackTrace(); + e.printStackTrace(originalOut); } finally { - if (bufferedReader != null) { - try { - bufferedReader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } + System.setOut(originalOut); } } From bc43b28ba6d0d1fa5339fd8d5494b982f1e88e26 Mon Sep 17 00:00:00 2001 From: svntax Date: Fri, 27 Jun 2025 06:51:08 -0400 Subject: [PATCH 15/19] Fix width for engine dropdown menu and remove unused import. --- src/main/java/com/example/gui/ChatboxViewBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 6f2b613..2fb26d7 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -21,7 +21,6 @@ import javafx.util.Builder; import java.io.File; -import java.nio.file.Paths; import java.util.function.Consumer; public class ChatboxViewBuilder implements Builder { @@ -71,6 +70,7 @@ private Node createEngineBox() { engineDropdown.valueProperty().bindBidirectional(model.selectedEngineProperty()); engineDropdown.getItems().addAll(ChatboxModel.Engine.values()); engineDropdown.setMaxWidth(Double.MAX_VALUE); + engineDropdown.setPrefWidth(152); HBox box = new HBox(8, createLabel("Engine:"), engineDropdown); box.setAlignment(Pos.CENTER_LEFT); return box; From d3814579924bb91c659ed7a4690dd648535d37ed Mon Sep 17 00:00:00 2001 From: svntax Date: Sat, 28 Jun 2025 02:11:02 -0400 Subject: [PATCH 16/19] Add dropdown menu for chat mode selection. --- .../java/com/example/gui/ChatboxInteractor.java | 6 ++++++ src/main/java/com/example/gui/ChatboxModel.java | 14 ++++++++++++++ .../java/com/example/gui/ChatboxViewBuilder.java | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index 29438fb..75c0cf3 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -28,6 +28,11 @@ private String[] buildCommands() { //commands.add("--gpu"); } + ChatboxModel.Mode mode = model.getSelectedMode(); + if (mode == ChatboxModel.Mode.INTERACTIVE) { + commands.add("--interactive"); + } + // Assume that models are found in a /models directory. String selectedModel = model.getSelectedModel(); if (selectedModel == null || selectedModel.isEmpty()) { @@ -91,6 +96,7 @@ private void process(String str) { Model llm = LlamaApp.loadModel(options); Sampler sampler = LlamaApp.createSampler(llm, options); if (options.interactive()) { + // TODO: how to read input from GUI text field? llm.runInteractive(sampler, options); } else { llm.runInstructOnce(sampler, options); diff --git a/src/main/java/com/example/gui/ChatboxModel.java b/src/main/java/com/example/gui/ChatboxModel.java index 727caf7..8d8d6a0 100644 --- a/src/main/java/com/example/gui/ChatboxModel.java +++ b/src/main/java/com/example/gui/ChatboxModel.java @@ -8,8 +8,10 @@ public class ChatboxModel { public enum Engine { TORNADO_VM, JVM } + public enum Mode { INSTRUCT, INTERACTIVE } private final ObjectProperty selectedEngine = new SimpleObjectProperty<>(Engine.TORNADO_VM); + private final ObjectProperty selectedMode = new SimpleObjectProperty<>(Mode.INSTRUCT); private final StringProperty selectedModel = new SimpleStringProperty(""); private final StringProperty promptText = new SimpleStringProperty(""); private final StringProperty outputText = new SimpleStringProperty(""); @@ -26,6 +28,18 @@ public void setSelectedEngine(Engine engine) { this.selectedEngine.set(engine); } + public Mode getSelectedMode() { + return selectedMode.get(); + } + + public ObjectProperty selectedModeProperty() { + return selectedMode; + } + + public void setSelectedMode(Mode mode) { + this.selectedMode.set(mode); + } + public String getSelectedModel() { return selectedModel.get(); } diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index 2fb26d7..e4f80b1 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -55,6 +55,7 @@ private Node createLeftPanel() { panel.getChildren().addAll( createHeaderLabel("TornadoVM Chat"), createEngineBox(), + createChatModeBox(), createModelSelectBox(), createLabel("Prompt:"), createPromptBox(), @@ -76,6 +77,17 @@ private Node createEngineBox() { return box; } + private Node createChatModeBox() { + ComboBox modeDropdown = new ComboBox<>(); + modeDropdown.valueProperty().bindBidirectional(model.selectedModeProperty()); + modeDropdown.getItems().addAll(ChatboxModel.Mode.values()); + modeDropdown.setMaxWidth(Double.MAX_VALUE); + modeDropdown.setPrefWidth(152); + HBox box = new HBox(8, createLabel("Chat:"), modeDropdown); + box.setAlignment(Pos.CENTER_LEFT); + return box; + } + private Node createModelSelectBox() { ComboBox modelDropdown = new ComboBox<>(); modelDropdown.valueProperty().bindBidirectional(model.selectedModelProperty()); From 69376025357f0f55f7c8e16f9307f186a52ee8d1 Mon Sep 17 00:00:00 2001 From: svntax Date: Fri, 4 Jul 2025 06:42:57 -0400 Subject: [PATCH 17/19] Change USE_TORNADOVM flag into a regular member variable. --- src/main/java/com/example/LlamaApp.java | 22 ++++++++++++++++++- .../com/example/gui/ChatboxInteractor.java | 6 +++-- .../example/loader/weights/ModelLoader.java | 3 ++- src/main/java/com/example/model/Model.java | 14 +++++++----- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/example/LlamaApp.java b/src/main/java/com/example/LlamaApp.java index 7c614a0..e739ca9 100644 --- a/src/main/java/com/example/LlamaApp.java +++ b/src/main/java/com/example/LlamaApp.java @@ -20,9 +20,29 @@ public class LlamaApp { // Configuration flags for hardware acceleration and optimizations public static final boolean USE_VECTOR_API = Boolean.parseBoolean(System.getProperty("llama.VectorAPI", "true")); // Enable Java Vector API for CPU acceleration public static final boolean USE_AOT = Boolean.parseBoolean(System.getProperty("llama.AOT", "false")); // Use Ahead-of-Time compilation - public static final boolean USE_TORNADOVM = Boolean.parseBoolean(System.getProperty("use.tornadovm", "false")); // Use TornadoVM for GPU acceleration + private boolean useTornadoVM = Boolean.parseBoolean(System.getProperty("use.tornadovm", "false")); // Use TornadoVM for GPU acceleration public static final boolean SHOW_PERF_INTERACTIVE = Boolean.parseBoolean(System.getProperty("llama.ShowPerfInteractive", "true")); // Show performance metrics in interactive mode + private static LlamaApp instance; + + private LlamaApp() { + } + + public static LlamaApp getInstance() { + if (instance == null) { + instance = new LlamaApp(); + } + return instance; + } + + public void setUseTornadoVM(boolean value) { + useTornadoVM = value; + } + + public boolean getUseTornadoVM() { + return useTornadoVM; + } + /** * Creates and configures a sampler for token generation based on specified parameters. * diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index 75c0cf3..7d1e6da 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -23,9 +23,11 @@ private String[] buildCommands() { List commands = new ArrayList<>(); ChatboxModel.Engine engine = model.getSelectedEngine(); + LlamaApp llamaApp = LlamaApp.getInstance(); if (engine == ChatboxModel.Engine.TORNADO_VM) { - // TODO: LlamaApp.USE_TORNADOVM is a final constant, but the GUI needs to be able to set this value - //commands.add("--gpu"); + llamaApp.setUseTornadoVM(true); + } else { + llamaApp.setUseTornadoVM(false); } ChatboxModel.Mode mode = model.getSelectedMode(); diff --git a/src/main/java/com/example/loader/weights/ModelLoader.java b/src/main/java/com/example/loader/weights/ModelLoader.java index 353600e..16cbb4d 100644 --- a/src/main/java/com/example/loader/weights/ModelLoader.java +++ b/src/main/java/com/example/loader/weights/ModelLoader.java @@ -96,7 +96,8 @@ public static Weights loadWeights(Map tensorEntries, Co GGMLTensorEntry tokenEmbeddings = tensorEntries.get("token_embd.weight"); GGMLTensorEntry outputWeight = tensorEntries.getOrDefault("output.weight", tokenEmbeddings); - if (LlamaApp.USE_TORNADOVM) { + LlamaApp llamaApp = LlamaApp.getInstance(); + if (llamaApp.getUseTornadoVM()) { System.out.println("Loading model weights in TornadoVM format (loading " + outputWeight.ggmlType() + " -> " + GGMLType.F16 + ")"); return createTornadoVMWeights(tensorEntries, config, ropeFreqs, tokenEmbeddings, outputWeight); } else { diff --git a/src/main/java/com/example/model/Model.java b/src/main/java/com/example/model/Model.java index e42349b..b31f112 100644 --- a/src/main/java/com/example/model/Model.java +++ b/src/main/java/com/example/model/Model.java @@ -1,5 +1,6 @@ package com.example.model; +import com.example.LlamaApp; import com.example.Options; import com.example.auxiliary.LastRunMetrics; import com.example.inference.InferenceEngine; @@ -17,7 +18,6 @@ import java.util.function.IntConsumer; import static com.example.LlamaApp.SHOW_PERF_INTERACTIVE; -import static com.example.LlamaApp.USE_TORNADOVM; public interface Model { Configuration configuration(); @@ -54,6 +54,9 @@ default void runInteractive(Sampler sampler, Options options) { // Initialize TornadoVM plan once at the beginning if GPU path is enabled TornadoVMMasterPlan tornadoVMPlan = null; + // Get the LlamaApp singleton to read configuration values + LlamaApp llamaApp = LlamaApp.getInstance(); + try { while (true) { System.out.print("> "); @@ -68,7 +71,7 @@ default void runInteractive(Sampler sampler, Options options) { state = createNewState(); } - if (USE_TORNADOVM && tornadoVMPlan == null) { + if (llamaApp.getUseTornadoVM() && tornadoVMPlan == null) { tornadoVMPlan = TornadoVMMasterPlan.initializeTornadoVMPlan(state, this); } @@ -86,7 +89,7 @@ default void runInteractive(Sampler sampler, Options options) { }; // Choose between GPU and CPU path based on configuration - if (USE_TORNADOVM) { + if (llamaApp.getUseTornadoVM()) { // GPU path using TornadoVM responseTokens = InferenceEngine.generateTokensGPU(this, state, startPosition, conversationTokens.subList(startPosition, conversationTokens.size()), stopTokens, options.maxTokens(), sampler, options.echo(), options.stream() ? tokenConsumer : null, tornadoVMPlan); @@ -121,7 +124,7 @@ default void runInteractive(Sampler sampler, Options options) { } } finally { // Clean up TornadoVM resources when exiting the chat loop - if (USE_TORNADOVM && tornadoVMPlan != null) { + if (llamaApp.getUseTornadoVM() && tornadoVMPlan != null) { try { tornadoVMPlan.freeTornadoExecutionPlan(); } catch (Exception e) { @@ -140,6 +143,7 @@ default void runInstructOnce(Sampler sampler, Options options) { State state = createNewState(); ChatFormat chatFormat = ChatFormat.create(tokenizer()); TornadoVMMasterPlan tornadoVMPlan = null; + LlamaApp llamaApp = LlamaApp.getInstance(); List promptTokens = new ArrayList<>(); promptTokens.add(chatFormat.getBeginOfText()); @@ -162,7 +166,7 @@ default void runInstructOnce(Sampler sampler, Options options) { Set stopTokens = chatFormat.getStopTokens(); - if (USE_TORNADOVM) { + if (llamaApp.getUseTornadoVM()) { tornadoVMPlan = TornadoVMMasterPlan.initializeTornadoVMPlan(state, this); // Call generateTokensGPU without the token consumer parameter responseTokens = InferenceEngine.generateTokensGPU(this, state, 0, promptTokens, stopTokens, options.maxTokens(), sampler, options.echo(), options.stream() ? tokenConsumer : null, From 2e2408f6195d5fdfc85e381bcbdba209fb4420bf Mon Sep 17 00:00:00 2001 From: svntax Date: Mon, 7 Jul 2025 04:32:25 -0400 Subject: [PATCH 18/19] Implement interactive mode for GUI chatbox. Created a new method `runInteractiveStep()` in Model.java for the GUI to use when interactive mode is selected. It uses a simple data object Model.Response, which keeps track of a chat session's conversation tokens, state, and TornadoVM plan while running. Every time the GUI runs this method, a new Response object is created with updated state, and ChatboxInteractor saves this response as part of the ongoing chat. --- .../com/example/gui/ChatboxInteractor.java | 79 ++++++++++++--- src/main/java/com/example/model/Model.java | 98 +++++++++++++++++++ 2 files changed, 163 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index 7d1e6da..c22b032 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -14,6 +14,10 @@ public class ChatboxInteractor { private final ChatboxModel model; + private Model llm; + private Sampler sampler; + private Model.Response currentResponse; + private boolean interactiveSessionActive = false; public ChatboxInteractor(ChatboxModel viewModel) { this.model = viewModel; @@ -44,14 +48,32 @@ private String[] buildCommands() { String modelPath = String.format("./models/%s", selectedModel); String prompt = String.format("\"%s\"", model.getPromptText()); - commands.addAll(Arrays.asList( - "--model", modelPath, - "--prompt", prompt - )); + commands.addAll(Arrays.asList("--model", modelPath)); + if (!interactiveSessionActive) { + commands.addAll(Arrays.asList("--prompt", prompt)); + } return commands.toArray(new String[commands.size()]); } + private void cleanTornadoVMResources() { + if (currentResponse != null && currentResponse.tornadoVMPlan() != null) { + try { + currentResponse.tornadoVMPlan().freeTornadoExecutionPlan(); + } catch (Exception e) { + System.err.println("Error while cleaning up TornadoVM resources: " + e.getMessage()); + } + } + } + + private void endInteractiveSession() { + cleanTornadoVMResources(); + llm = null; + currentResponse = null; + interactiveSessionActive = false; + System.out.println("Interactive session ended"); + } + // Load and run a model while capturing its output text to a custom stream. public void runLlamaTornado() { // Save the original System.out stream @@ -65,9 +87,6 @@ public void runLlamaTornado() { StringBuilder builder = new StringBuilder(); - builder.append("Processing... please wait"); - builder.append(System.getProperty("line.separator")); - // Create a custom PrintStream to capture output from loading the model and running it. PrintStream customStream = new PrintStream(new ByteArrayOutputStream()) { @Override @@ -94,19 +113,51 @@ private void process(String str) { Options options = Options.parseOptions(commands); - // Load the model and run. - Model llm = LlamaApp.loadModel(options); - Sampler sampler = LlamaApp.createSampler(llm, options); - if (options.interactive()) { - // TODO: how to read input from GUI text field? - llm.runInteractive(sampler, options); + if (interactiveSessionActive) { + builder.append(model.getOutputText()); // Include the current output to avoid clearing the entire text. + String userText = model.getPromptText(); + // Display the user message with a '>' prefix + builder.append("> "); + builder.append(userText); + builder.append(System.getProperty("line.separator")); + if (List.of("quit", "exit").contains(userText)) { + endInteractiveSession(); + } else { + currentResponse = llm.runInteractiveStep(sampler, options, userText, currentResponse); + } } else { - llm.runInstructOnce(sampler, options); + builder.append("Processing... please wait"); + builder.append(System.getProperty("line.separator")); + + // Load the model and run. + llm = LlamaApp.loadModel(options); + sampler = LlamaApp.createSampler(llm, options); + if (options.interactive()) { + // Start a new interactive session. + // TODO: disable the rest of the GUI while a session is active + builder.append("Interactive session started (write 'exit' or 'quit' to stop)"); + builder.append(System.getProperty("line.separator")); + // Display the user message with a '>' prefix + builder.append("> "); + builder.append(model.getPromptText()); + builder.append(System.getProperty("line.separator")); + currentResponse = llm.runInteractiveStep(sampler, options, model.getPromptText(), new Model.Response()); + interactiveSessionActive = true; + } else { + llm.runInstructOnce(sampler, options); + llm = null; + sampler = null; + } } + } catch (Exception e) { // Catch all exceptions so that they're logged in the output area. e.printStackTrace(); e.printStackTrace(originalOut); + // Make sure to end the interactive session if an exception occurs. + if (interactiveSessionActive) { + endInteractiveSession(); + } } finally { System.setOut(originalOut); } diff --git a/src/main/java/com/example/model/Model.java b/src/main/java/com/example/model/Model.java index b31f112..bbf2a46 100644 --- a/src/main/java/com/example/model/Model.java +++ b/src/main/java/com/example/model/Model.java @@ -134,6 +134,104 @@ default void runInteractive(Sampler sampler, Options options) { } } + /** + * Model agnostic implementation for interactive GUI mode. + * Takes a single user input and returns the model's response, allowing the GUI to manage the chat loop. + * + * @param sampler The sampler for token generation + * @param options The inference options + * @param userText The user's input text + * @param previousResponse A Response object referencing an ongoing chat + * @return A Response object containing the model's output and updated state + */ + default Response runInteractiveStep(Sampler sampler, Options options, String userText, Response previousResponse) { + ChatFormat chatFormat = ChatFormat.create(tokenizer()); + List conversationTokens = previousResponse.conversationTokens(); + State state = previousResponse.state(); + TornadoVMMasterPlan tornadoVMPlan = previousResponse.tornadoVMPlan(); + String responseText = ""; + + int startPosition = conversationTokens.size(); + + // For the first message, set up the conversation tokens if empty + if (conversationTokens.isEmpty()) { + conversationTokens.add(chatFormat.getBeginOfText()); + if (options.systemPrompt() != null) { + conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.SYSTEM, options.systemPrompt()))); + } + } + + // Get the LlamaApp singleton to read configuration values + LlamaApp llamaApp = LlamaApp.getInstance(); + + if (state == null) { + // State allocation can take some time for large context sizes + state = createNewState(); + } + + // Initialize TornadoVM plan once at the beginning if GPU path is enabled + if (llamaApp.getUseTornadoVM() && tornadoVMPlan == null) { + tornadoVMPlan = TornadoVMMasterPlan.initializeTornadoVMPlan(state, this); + } + + conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.USER, userText))); + conversationTokens.addAll(chatFormat.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, ""))); + Set stopTokens = chatFormat.getStopTokens(); + + List responseTokens; + IntConsumer tokenConsumer = token -> { + if (options.stream()) { + if (tokenizer().shouldDisplayToken(token)) { + System.out.print(tokenizer().decode(List.of(token))); + } + } + }; + + // Choose between GPU and CPU path based on configuration + if (llamaApp.getUseTornadoVM()) { + // GPU path using TornadoVM + responseTokens = InferenceEngine.generateTokensGPU(this, state, startPosition, conversationTokens.subList(startPosition, conversationTokens.size()), stopTokens, + options.maxTokens(), sampler, options.echo(), options.stream() ? tokenConsumer : null, tornadoVMPlan); + } else { + // CPU path + responseTokens = InferenceEngine.generateTokens(this, state, startPosition, conversationTokens.subList(startPosition, conversationTokens.size()), stopTokens, options.maxTokens(), + sampler, options.echo(), tokenConsumer); + } + + // Include stop token in the prompt history, but not in the response displayed to the user. + conversationTokens.addAll(responseTokens); + Integer stopToken = null; + if (!responseTokens.isEmpty() && stopTokens.contains(responseTokens.getLast())) { + stopToken = responseTokens.getLast(); + responseTokens.removeLast(); + } + if (!options.stream()) { + responseText = tokenizer().decode(responseTokens); + System.out.println(responseText); + } + if (stopToken == null) { + System.err.println("\n Ran out of context length...\n Increase context length by passing to llama-tornado --max-tokens XXX"); + return new Response(responseText, state, conversationTokens, tornadoVMPlan); + } + System.out.print("\n"); + + // Optionally print performance metrics after each response + if (SHOW_PERF_INTERACTIVE) { + LastRunMetrics.printMetrics(); + } + + return new Response(responseText, state, conversationTokens, tornadoVMPlan); + } + + /** + * Simple data model for Model responses, used to keep track of conversation history and state. + */ + record Response(String responseText, State state, List conversationTokens, TornadoVMMasterPlan tornadoVMPlan) { + public Response() { + this("", null, new ArrayList<>(), null); + } + } + /** * Model agnostic default implementation for instruct mode. * @param sampler From 480336e93b8d0ab0069123b035a4cacc8098c257 Mon Sep 17 00:00:00 2001 From: svntax Date: Mon, 7 Jul 2025 05:34:18 -0400 Subject: [PATCH 19/19] Disable GUI controls while an interactive chat is running. Prevent the user from changing any of the GUI settings (like the model and engine) while an interactive session is running. --- .../java/com/example/gui/ChatboxInteractor.java | 12 +++++------- src/main/java/com/example/gui/ChatboxModel.java | 15 +++++++++++++++ .../java/com/example/gui/ChatboxViewBuilder.java | 14 +++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/example/gui/ChatboxInteractor.java b/src/main/java/com/example/gui/ChatboxInteractor.java index c22b032..b740fd8 100644 --- a/src/main/java/com/example/gui/ChatboxInteractor.java +++ b/src/main/java/com/example/gui/ChatboxInteractor.java @@ -17,7 +17,6 @@ public class ChatboxInteractor { private Model llm; private Sampler sampler; private Model.Response currentResponse; - private boolean interactiveSessionActive = false; public ChatboxInteractor(ChatboxModel viewModel) { this.model = viewModel; @@ -49,7 +48,7 @@ private String[] buildCommands() { String prompt = String.format("\"%s\"", model.getPromptText()); commands.addAll(Arrays.asList("--model", modelPath)); - if (!interactiveSessionActive) { + if (!model.getInteractiveSessionActive()) { commands.addAll(Arrays.asList("--prompt", prompt)); } @@ -70,7 +69,7 @@ private void endInteractiveSession() { cleanTornadoVMResources(); llm = null; currentResponse = null; - interactiveSessionActive = false; + model.setInteractiveSessionActive(false); System.out.println("Interactive session ended"); } @@ -113,7 +112,7 @@ private void process(String str) { Options options = Options.parseOptions(commands); - if (interactiveSessionActive) { + if (model.getInteractiveSessionActive()) { builder.append(model.getOutputText()); // Include the current output to avoid clearing the entire text. String userText = model.getPromptText(); // Display the user message with a '>' prefix @@ -134,7 +133,6 @@ private void process(String str) { sampler = LlamaApp.createSampler(llm, options); if (options.interactive()) { // Start a new interactive session. - // TODO: disable the rest of the GUI while a session is active builder.append("Interactive session started (write 'exit' or 'quit' to stop)"); builder.append(System.getProperty("line.separator")); // Display the user message with a '>' prefix @@ -142,7 +140,7 @@ private void process(String str) { builder.append(model.getPromptText()); builder.append(System.getProperty("line.separator")); currentResponse = llm.runInteractiveStep(sampler, options, model.getPromptText(), new Model.Response()); - interactiveSessionActive = true; + model.setInteractiveSessionActive(true); } else { llm.runInstructOnce(sampler, options); llm = null; @@ -155,7 +153,7 @@ private void process(String str) { e.printStackTrace(); e.printStackTrace(originalOut); // Make sure to end the interactive session if an exception occurs. - if (interactiveSessionActive) { + if (model.getInteractiveSessionActive()) { endInteractiveSession(); } } finally { diff --git a/src/main/java/com/example/gui/ChatboxModel.java b/src/main/java/com/example/gui/ChatboxModel.java index 8d8d6a0..16a4cd4 100644 --- a/src/main/java/com/example/gui/ChatboxModel.java +++ b/src/main/java/com/example/gui/ChatboxModel.java @@ -1,6 +1,8 @@ package com.example.gui; +import javafx.beans.property.BooleanProperty; import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.beans.property.SimpleStringProperty; import javafx.beans.property.StringProperty; @@ -15,6 +17,7 @@ public enum Mode { INSTRUCT, INTERACTIVE } private final StringProperty selectedModel = new SimpleStringProperty(""); private final StringProperty promptText = new SimpleStringProperty(""); private final StringProperty outputText = new SimpleStringProperty(""); + private final BooleanProperty interactiveSessionActive = new SimpleBooleanProperty(false); public Engine getSelectedEngine() { return selectedEngine.get(); @@ -76,4 +79,16 @@ public void setOutputText(String text) { this.outputText.set(text); } + public boolean getInteractiveSessionActive() { + return interactiveSessionActive.get(); + } + + public BooleanProperty interactiveSessionActiveProperty() { + return interactiveSessionActive; + } + + public void setInteractiveSessionActive(boolean value) { + this.interactiveSessionActive.set(value); + } + } \ No newline at end of file diff --git a/src/main/java/com/example/gui/ChatboxViewBuilder.java b/src/main/java/com/example/gui/ChatboxViewBuilder.java index e4f80b1..ee6894a 100644 --- a/src/main/java/com/example/gui/ChatboxViewBuilder.java +++ b/src/main/java/com/example/gui/ChatboxViewBuilder.java @@ -1,6 +1,7 @@ package com.example.gui; import atlantafx.base.theme.Styles; +import javafx.beans.binding.Bindings; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.StringProperty; @@ -68,6 +69,9 @@ private Node createLeftPanel() { private Node createEngineBox() { ComboBox engineDropdown = new ComboBox<>(); + engineDropdown.disableProperty().bind(Bindings.createBooleanBinding(() -> (inferenceRunning.get() || model.getInteractiveSessionActive()), + inferenceRunning, + model.interactiveSessionActiveProperty())); engineDropdown.valueProperty().bindBidirectional(model.selectedEngineProperty()); engineDropdown.getItems().addAll(ChatboxModel.Engine.values()); engineDropdown.setMaxWidth(Double.MAX_VALUE); @@ -79,6 +83,9 @@ private Node createEngineBox() { private Node createChatModeBox() { ComboBox modeDropdown = new ComboBox<>(); + modeDropdown.disableProperty().bind(Bindings.createBooleanBinding(() -> (inferenceRunning.get() || model.getInteractiveSessionActive()), + inferenceRunning, + model.interactiveSessionActiveProperty())); modeDropdown.valueProperty().bindBidirectional(model.selectedModeProperty()); modeDropdown.getItems().addAll(ChatboxModel.Mode.values()); modeDropdown.setMaxWidth(Double.MAX_VALUE); @@ -90,6 +97,9 @@ private Node createChatModeBox() { private Node createModelSelectBox() { ComboBox modelDropdown = new ComboBox<>(); + modelDropdown.disableProperty().bind(Bindings.createBooleanBinding(() -> (inferenceRunning.get() || model.getInteractiveSessionActive()), + inferenceRunning, + model.interactiveSessionActiveProperty())); modelDropdown.valueProperty().bindBidirectional(model.selectedModelProperty()); HBox.setHgrow(modelDropdown, Priority.ALWAYS); modelDropdown.setMaxWidth(Double.MAX_VALUE); @@ -97,7 +107,9 @@ private Node createModelSelectBox() { Button reloadButton = new Button("Reload"); reloadButton.getStyleClass().add(Styles.ACCENT); reloadButton.setMinWidth(80); - reloadButton.disableProperty().bind(inferenceRunning); + reloadButton.disableProperty().bind(Bindings.createBooleanBinding(() -> (inferenceRunning.get() || model.getInteractiveSessionActive()), + inferenceRunning, + model.interactiveSessionActiveProperty())); reloadButton.setOnAction(e -> { // Search for a /models folder containing model files. modelDropdown.getItems().clear();