diff --git a/build.gradle b/build.gradle index c5d3835..26de2cf 100644 --- a/build.gradle +++ b/build.gradle @@ -49,6 +49,7 @@ dependencies { shadow libs.qupath.fxtras shadow libs.bioimageio.spec shadow libs.deepJavaLibrary + shadow libs.commonmark implementation 'io.github.qupath:qupath-extension-djl:0.3.0' diff --git a/src/main/java/qupath/ext/instanseg/core/DetectionMeasurer.java b/src/main/java/qupath/ext/instanseg/core/DetectionMeasurer.java index cfa0184..44c48b4 100644 --- a/src/main/java/qupath/ext/instanseg/core/DetectionMeasurer.java +++ b/src/main/java/qupath/ext/instanseg/core/DetectionMeasurer.java @@ -28,9 +28,9 @@ private DetectionMeasurer(Collection compartmen Collection shapeFeatures, double downsample) { this.shapeFeatures = shapeFeatures; - this.downsample = downsample; this.compartments = compartments; this.measurements = measurements; + this.downsample = downsample; } /** diff --git a/src/main/java/qupath/ext/instanseg/core/InstanSeg.java b/src/main/java/qupath/ext/instanseg/core/InstanSeg.java index d726a37..d8cb8f2 100644 --- a/src/main/java/qupath/ext/instanseg/core/InstanSeg.java +++ b/src/main/java/qupath/ext/instanseg/core/InstanSeg.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.stream.IntStream; @@ -136,8 +137,11 @@ private InstanSegResults runInstanSeg(Collection pathObjec long startTime = System.currentTimeMillis(); - Path modelPath; - modelPath = model.getPath().resolve("instanseg.pt"); + Optional oModelPath = model.getPath(); + if (!oModelPath.isPresent()) { + return new InstanSegResults(0, 0, 0, 0, 0); + } + Path modelPath = oModelPath.get().resolve("instanseg.pt"); int nPredictors = 1; // todo: change me? // Optionally pad images so that every tile has the required size. @@ -162,7 +166,7 @@ private InstanSegResults runInstanSeg(Collection pathObjec // Create an int[] representing a boolean array of channels to use boolean[] outputChannelArray = null; if (outputChannels != null && outputChannels.length > 0) { - outputChannelArray = new boolean[model.getOutputChannels()];; + outputChannelArray = new boolean[model.getOutputChannels().get()]; // safe to call get because of previous checks for (int c : outputChannels) { if (c < 0 || c >= outputChannelArray.length) { throw new IllegalArgumentException("Invalid channel index: " + c); @@ -514,15 +518,6 @@ public Builder modelPath(String path) throws IOException { return modelPath(Path.of(path)); } - /** - * Set the specific model to be used - * @param name The name of a built-in model - * @return A modified builder - */ - public Builder modelName(String name) { - return model(InstanSegModel.fromName(name)); - } - /** * Set the device to be used * @param deviceName The name of the device to be used (eg, "gpu", "mps"). diff --git a/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java b/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java index 2331784..aa7c4a2 100644 --- a/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java +++ b/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java @@ -3,22 +3,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import qupath.bioimageio.spec.BioimageIoSpec; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; import qupath.lib.common.GeneralTools; import qupath.lib.gui.UserDirectoryManager; import qupath.lib.images.servers.PixelCalibration; import java.io.IOException; +import java.io.InputStream; import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import java.util.Objects; public class InstanSegModel { private static final Logger logger = LoggerFactory.getLogger(InstanSegModel.class); + private URL modelURL = null; /** * Constant to indicate that any number of channels are supported. @@ -26,17 +39,16 @@ public class InstanSegModel { public static final int ANY_CHANNELS = -1; private Path path = null; - private URL modelURL = null; private BioimageIoSpec.BioimageIoModel model = null; private final String name; private InstanSegModel(BioimageIoSpec.BioimageIoModel bioimageIoModel) { this.model = bioimageIoModel; this.path = Paths.get(model.getBaseURI()); - this.name = model.getName(); + this.name = model.getName() + " (local)"; } - private InstanSegModel(URL modelURL, String name) { + private InstanSegModel(String name, URL modelURL) { this.modelURL = modelURL; this.name = name; } @@ -52,46 +64,85 @@ public static InstanSegModel fromPath(Path path) throws IOException { } /** - * Request an InstanSeg model from the set of available models + * Create an InstanSeg model from a remote URL. * @param name The model name - * @return The specified model. + * @param browserDownloadUrl The download URL from eg GitHub + * @return A handle on the created model + */ + public static InstanSegModel fromURL(String name, URL browserDownloadUrl) { + return new InstanSegModel(name, browserDownloadUrl); + } + + /** + * Check if the model has been downloaded already. + * @return True if a flag has been set. */ - public static InstanSegModel fromName(String name) { - // todo: instantiate built-in models somehow - throw new UnsupportedOperationException("Fetching models by name is not yet implemented!"); + public boolean isDownloaded(Path localModelPath) { + // todo: this should also check if the contents are what we expect + if (Files.exists(localModelPath.resolve(name))) { + try { + download(localModelPath); + } catch (IOException e) { + logger.error("Model directory exists but is not valid", e); + } + } + return path != null && model != null; + } + + /** + * Trigger a download for a model + * @throws IOException If an error occurs when downloading, unzipping, etc. + */ + public void download(Path localModelPath) throws IOException { + if (path != null && model != null) return; + var zipFile = downloadZipIfNeeded( + this.modelURL, + localModelPath, + name); + this.path = unzipIfNeeded(zipFile); + this.model = BioimageIoSpec.parseModel(path.toFile()); + } + + /** + * Extract the README from a local file + * @return The README as a String, if possible. If not present, or an error + * occurs when reading, nothing. + */ + public Optional getREADME() { + return getPath().map(this::getREADMEString); } /** * Get the pixel size in the X dimension. - * @return the pixel size in the X dimension. + * @return the pixel size in the X dimension, or empty if the model isn't downloaded yet. */ - private Number getPixelSizeX() { - return getPixelSize().getOrDefault("x", null); + private Optional getPixelSizeX() { + return getPixelSize().flatMap(p -> Optional.ofNullable(p.getOrDefault("x", null))); } /** * Get the pixel size in the Y dimension. - * @return the pixel size in the Y dimension. + * @return the pixel size in the Y dimension, or empty if the model isn't downloaded yet. */ - private Number getPixelSizeY() { - return getPixelSize().getOrDefault("y", null); + private Optional getPixelSizeY() { + return getPixelSize().flatMap(p -> Optional.ofNullable(p.getOrDefault("y", null))); } /** * Get the preferred pixel size for running the model, in the absence of any other information. * This is the average of the X and Y pixel sizes if both are available. * Otherwise, it is the value of whichever value is available - or null if neither is found. - * @return the pixel size + * @return the pixel size, or empty if the model isn't downloaded yet. */ - public Number getPreferredPixelSize() { + public Optional getPreferredPixelSize() { var x = getPixelSizeX(); var y = getPixelSizeY(); - if (x == null) { + if (x.isEmpty()) { return y; - } else if (y == null) { + } else if (y.isEmpty()) { return x; } else { - return (x.doubleValue() + y.doubleValue()) / 2.0; + return Optional.of((x.get().doubleValue() + y.get().doubleValue()) / 2.0); } } @@ -113,12 +164,12 @@ public Number getPreferredPixelSize() { */ public double getPreferredDownsample(PixelCalibration cal) { Objects.requireNonNull(cal, "Pixel calibration must not be null"); - Number preferred = getPreferredPixelSize(); + Optional preferred = getPreferredPixelSize(); double current = cal.getAveragedPixelSize().doubleValue(); - if (preferred == null) { + if (preferred.isEmpty()) { return current; } - double requested = preferred.doubleValue(); + double requested = preferred.get().doubleValue(); if (requested > 0 && current > 0) { return getPreferredDownsample(current, requested); } else { @@ -130,9 +181,9 @@ public double getPreferredDownsample(PixelCalibration cal) { /** * Get the preferred pixel size for running the model, incorporating information from the pixel calibration of the * image. This tries to encourage downsampling by an integer amount. - * @param currentPixelSize - * @param requestedPixelSize - * @return + * @param currentPixelSize The current pixel size (probably in microns) + * @param requestedPixelSize The pixel size that the model expects (probably in microns) + * @return The exact downsample, unless it's close to an integer, in which case the integer. */ static double getPreferredDownsample(double currentPixelSize, double requestedPixelSize) { double downsample = requestedPixelSize / currentPixelSize; @@ -146,13 +197,10 @@ static double getPreferredDownsample(double currentPixelSize, double requestedPi /** * Get the path where the model is stored on disk. - * @return A path on disk, or an exception if it can't be found. + * @return A path on disk. */ - public Path getPath() { - if (path == null) { - fetchModel(); - } - return path; + public Optional getPath() { + return Optional.ofNullable(path); } @Override @@ -168,7 +216,6 @@ public String toString() { * the yaml contents and the checksum of the pt file. */ public static boolean isValidModel(Path path) { - // return path.toString().endsWith(".pt"); // if just looking at pt files if (Files.isDirectory(path)) { return Files.exists(path.resolve("instanseg.pt")) && Files.exists(path.resolve("rdf.yaml")); } @@ -179,42 +226,22 @@ public static boolean isValidModel(Path path) { * Get the model name * @return A string */ - String getName() { + public String getName() { return name; } + /** - * Retrieve the BioImage model spec. - * @return The BioImageIO model spec for this InstanSeg model. + * Try to check the number of channels in the model. + * @return The integer if the model is downloaded, otherwise empty */ - private BioimageIoSpec.BioimageIoModel getModel() { - if (model == null) { - fetchModel(); - } - return model; - } - - private Map getPixelSize() { - // todo: this code is horrendous - var config = getModel().getConfig().getOrDefault("qupath", null); - if (config instanceof Map configMap) { - var axes = (List)configMap.get("axes"); - return Map.of( - "x", (Double) ((Map) (axes.get(0))).get("step"), - "y", (Double) ((Map) (axes.get(1))).get("step") - ); - } - return Map.of("x", 1.0, "y", 1.0); + public Optional getNumChannels() { + return getModel().flatMap(model -> Optional.of(extractChannelNum(model))); } - /** - * Get the number of input channels supported by the model. - * @return a positive integer, or {@link #ANY_CHANNELS} if any number of channels is supported. - */ - public int getInputChannels() { - String axes = getModel().getInputs().getFirst().getAxes().toLowerCase(); - int ind = axes.indexOf("c"); - var shape = getModel().getInputs().getFirst().getShape(); + private static int extractChannelNum(BioimageIoSpec.BioimageIoModel model) { + int ind = model.getInputs().getFirst().getAxes().toLowerCase().indexOf("c"); + var shape = model.getInputs().getFirst().getShape(); if (shape.getShapeStep()[ind] == 1) { return ANY_CHANNELS; } else { @@ -222,36 +249,129 @@ public int getInputChannels() { } } + /** - * Get the number of output channels provided by the model (typically 1 or 2) - * @return a positive integer + * Retrieve the BioImage model spec. + * @return The BioImageIO model spec for this InstanSeg model. */ - public int getOutputChannels() { - var output = getModel().getOutputs().getFirst(); - String axes = output.getAxes().toLowerCase(); - int ind = axes.indexOf("c"); - var shape = output.getShape().getShape(); - if (shape != null && shape.length > ind) - return shape[ind]; - return (int)Math.round(output.getShape().getOffset()[ind] * 2); + private Optional getModel() { + return Optional.ofNullable(model); + } + + + private static Path downloadZipIfNeeded(URL url, Path localDirectory, String filename) throws IOException { + var zipFile = localDirectory.resolve(Path.of(filename + ".zip")); + if (!isDownloadedAlready(zipFile)) { + try (InputStream stream = url.openStream()) { + try (ReadableByteChannel readableByteChannel = Channels.newChannel(stream)) { + try (FileOutputStream fos = new FileOutputStream(zipFile.toFile())) { + fos.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE); + } + } + } + } + return zipFile; + } + + private static boolean isDownloadedAlready(Path zipFile) { + // todo: validate contents somehow + return Files.exists(zipFile); + } + + private static Path unzipIfNeeded(Path zipFile) throws IOException { + var outdir = zipFile.resolveSibling(zipFile.getFileName().toString().replace(".zip", "")); + if (!isUnpackedAlready(outdir)) { + try { + unzip(zipFile, zipFile.getParent()); + // Files.delete(zipFile); + } catch (IOException e) { + // clean up files just in case! + Files.deleteIfExists(zipFile); + Files.deleteIfExists(outdir); + } + } + return outdir; + } + + private static boolean isUnpackedAlready(Path outdir) { + return Files.exists(outdir) && isValidModel(outdir); + } + + private static void unzip(Path zipFile, Path destination) throws IOException { + if (!Files.exists(destination)) { + Files.createDirectory(destination); + } + ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile.toFile()))); + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + Path filePath = destination.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectory(filePath); + } else { + extractFile(zipIn, filePath); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + zipIn.close(); } - private void fetchModel() { - if (modelURL == null) { - throw new NullPointerException("Model URL should not be null for a local model!"); + private static void extractFile(ZipInputStream zipIn, Path filePath) throws IOException { + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath.toFile())); + byte[] bytesIn = new byte[4096]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + bos.close(); + } + + + + private String getREADMEString(Path path) { + var file = path.resolve(name + "_README.md"); + if (Files.exists(file)) { + try { + return Files.readString(file, StandardCharsets.UTF_8); + } catch (IOException e) { + logger.error("Unable to find README", e); + return null; + } + } else { + logger.debug("No README found for model {}", name); + return null; } - downloadAndUnzip(modelURL, getUserDir().resolve("instanseg")); } - private static void downloadAndUnzip(URL url, Path localDirectory) { - // todo: implement - throw new UnsupportedOperationException("Downloading and unzipping models is not yet implemented!"); + private Optional> getPixelSize() { + return getModel().flatMap(model -> { + var config = model.getConfig().getOrDefault("qupath", null); + if (config instanceof Map configMap) { + var axes = (List) configMap.get("axes"); + return Optional.of(Map.of( + "x", (Double) ((Map) (axes.get(0))).get("step"), + "y", (Double) ((Map) (axes.get(1))).get("step") + )); + } + return Optional.of(Map.of("x", 1.0, "y", 1.0)); + }); } - private static Path getUserDir() { - Path userPath = UserDirectoryManager.getInstance().getUserPath(); - Path cachePath = Paths.get(System.getProperty("user.dir"), ".cache", "QuPath"); - return userPath == null || userPath.toString().isEmpty() ? cachePath : userPath; + /** + * Get the number of output channels provided by the model (typically 1 or 2) + * @return a positive integer + */ + public Optional getOutputChannels() { + return getModel().map(model -> { + var output = model.getOutputs().getFirst(); + String axes = output.getAxes().toLowerCase(); + int ind = axes.indexOf("c"); + var shape = output.getShape().getShape(); + if (shape != null && shape.length > ind) + return shape[ind]; + return (int)Math.round(output.getShape().getOffset()[ind] * 2); + }); } + } diff --git a/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java b/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java index aeb3b06..3a7246c 100644 --- a/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java +++ b/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java @@ -1,8 +1,16 @@ package qupath.ext.instanseg.ui; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; import javafx.application.Platform; import javafx.beans.binding.Bindings; +import javafx.beans.binding.BooleanBinding; +import javafx.beans.binding.ObjectBinding; +import javafx.beans.property.BooleanProperty; import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.beans.property.StringProperty; import javafx.collections.FXCollections; @@ -23,7 +31,10 @@ import javafx.scene.control.ToggleButton; import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; +import javafx.scene.web.WebView; +import org.commonmark.renderer.html.HtmlRenderer; import org.controlsfx.control.CheckComboBox; +import org.controlsfx.control.PopOver; import org.controlsfx.control.SearchableComboBox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +50,7 @@ import qupath.lib.gui.QuPathGUI; import qupath.lib.gui.TaskRunnerFX; import qupath.lib.gui.tools.GuiTools; +import qupath.lib.gui.tools.WebViews; import qupath.lib.images.ImageData; import qupath.lib.images.servers.ImageServer; import qupath.lib.plugins.workflow.DefaultScriptableWorkflowStep; @@ -46,18 +58,26 @@ import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.ResourceBundle; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.FutureTask; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -66,8 +86,11 @@ * Controller for UI pane contained in instanseg_control.fxml */ public class InstanSegController extends BorderPane { + private static final Logger logger = LoggerFactory.getLogger(InstanSegController.class); + private static final ResourceBundle resources = ResourceBundle.getBundle("qupath.ext.instanseg.ui.strings"); + private final Watcher watcher; @FXML private CheckComboBox comboChannels; @@ -78,6 +101,8 @@ public class InstanSegController extends BorderPane { @FXML private Button runButton; @FXML + private Button downloadButton; + @FXML private Label labelMessage; @FXML private ChoiceBox deviceChoices; @@ -93,15 +118,37 @@ public class InstanSegController extends BorderPane { private CheckBox nucleiOnlyCheckBox; @FXML private CheckBox makeMeasurementsCheckBox; + @FXML + private Button infoButton; + @FXML + private Label modelDirLabel; private final ExecutorService pool = Executors.newSingleThreadExecutor(ThreadTools.createThreadFactory("instanseg", true)); private final QuPathGUI qupath; - private ObjectProperty> pendingTask = new SimpleObjectProperty<>(); + private final ObjectProperty> pendingTask = new SimpleObjectProperty<>(); private MessageTextHelper messageTextHelper; - private final Watcher watcher = new Watcher(modelChoiceBox); - private ExecutorService executor; + private final BooleanProperty needsUpdating = new SimpleBooleanProperty(); + + private static final ObjectBinding modelDirectoryProperty = Bindings.createObjectBinding( + () -> tryToGetPath(InstanSegPreferences.modelDirectoryProperty().get()), + InstanSegPreferences.modelDirectoryProperty() + ); + private final BooleanBinding isModelDirectoryValid = Bindings.createBooleanBinding( + () -> { + var path = modelDirectoryProperty.get(); + return path != null && Files.isDirectory(path); + }, + modelDirectoryProperty + ); + + /** + * Create an instance of the InstanSeg GUI pane. + * @param qupath The QuPath GUI it should be attached to. + * @return A handle on the UI element. + * @throws IOException If the FXML or resources fail to load. + */ public static InstanSegController createInstance(QuPathGUI qupath) throws IOException { return new InstanSegController(qupath); } @@ -113,18 +160,39 @@ private InstanSegController(QuPathGUI qupath) throws IOException { loader.setRoot(this); loader.setController(this); loader.load(); + watcher = new Watcher(modelChoiceBox); configureMessageLabel(); + configureDirectoryLabel(); configureTileSizes(); configureDeviceChoices(); configureModelChoices(); configureSelectButtons(); configureRunning(); configureThreadSpinner(); - + BooleanBinding currentModelIsDownloaded = createModelDownloadedBinding(); + infoButton.disableProperty().bind(currentModelIsDownloaded.not()); + downloadButton.disableProperty().bind( + currentModelIsDownloaded.or( + modelChoiceBox.getSelectionModel().selectedItemProperty().isNull()) + ); configureChannelPicker(); } + private BooleanBinding createModelDownloadedBinding() { + return Bindings.createBooleanBinding( + () -> { + var model = modelChoiceBox.getSelectionModel().getSelectedItem(); + if (model == null) { + return false; + } + var modelDir = getModelDirectory().orElse(null); + return modelDir != null && model.isDownloaded(modelDir); + }, + modelChoiceBox.getSelectionModel().selectedItemProperty(), needsUpdating, + InstanSegPreferences.modelDirectoryProperty()); + } + void interrupt() { watcher.interrupt(); @@ -139,15 +207,14 @@ void handleModelDirectoryLabelClick(MouseEvent event) { if (event.getClickCount() != 2) { return; } - var path = InstanSegPreferences.modelDirectoryProperty().get(); - if (path == null || path.isEmpty()) { + var modelDir = getModelDirectory().orElse(null); + if (modelDir == null) { return; } - var file = new File(path); - if (file.exists()) { - GuiTools.browseDirectory(file); + if (Files.exists(modelDir)) { + GuiTools.browseDirectory(modelDir.toFile()); } else { - logger.debug("Can't browse directory for {}", file); + logger.debug("Can't browse directory for {}", modelDir); } } @@ -182,9 +249,16 @@ private void updateChannelPicker(ImageData imageData) { if (imageData.isBrightfield()) { comboChannels.getCheckModel().checkIndices(IntStream.range(0, 3).toArray()); var model = modelChoiceBox.getSelectionModel().selectedItemProperty().get(); - if (model != null && model.getInputChannels() != InstanSegModel.ANY_CHANNELS) { - comboChannels.getCheckModel().clearChecks(); - comboChannels.getCheckModel().checkIndices(0, 1, 2); + if (model != null && model.isDownloaded(Path.of(InstanSegPreferences.modelDirectoryProperty().get()))) { + var modelChannels = model.getNumChannels(); + if (modelChannels.isPresent()) { + int nModelChannels = modelChannels.get(); + if (nModelChannels != InstanSegModel.ANY_CHANNELS) { + comboChannels.getCheckModel().clearChecks(); + comboChannels.getCheckModel().checkIndices(0, 1, 2); + } + } + } } else { comboChannels.getCheckModel().checkIndices(IntStream.range(0, imageData.getServer().nChannels()).toArray()); @@ -288,7 +362,6 @@ private void configureRunning() { runButton.disableProperty().bind( qupath.imageDataProperty().isNull() .or(pendingTask.isNotNull()) - .or(modelChoiceBox.getSelectionModel().selectedItemProperty().isNull()) .or(messageTextHelper.hasWarning()) .or(deviceChoices.getSelectionModel().selectedItemProperty().isNull()) .or(Bindings.createBooleanBinding(() -> { @@ -296,10 +369,15 @@ private void configureRunning() { if (model == null) { return true; } - int numSelected = comboChannels.getCheckModel().getCheckedIndices().size(); - int numAllowed = model.getInputChannels(); - return !(numSelected == numAllowed || numAllowed == InstanSegModel.ANY_CHANNELS); - }, modelChoiceBox.getSelectionModel().selectedItemProperty())) + var modelDir = getModelDirectory().orElse(null); + if (modelDir == null || !Files.exists(modelDir)) { + return true; // Can't download without somewhere to put it + } + if (!model.isDownloaded(modelDir)) { + return false; // to enable "download and run" + } + return false; + }, modelChoiceBox.getSelectionModel().selectedItemProperty(), needsUpdating)) ); pendingTask.addListener((observable, oldValue, newValue) -> { if (newValue != null) { @@ -308,24 +386,115 @@ private void configureRunning() { }); } + private static Path tryToGetPath(String path) { + if (path == null || path.isEmpty()) { + return null; + } else { + return Path.of(path); + } + } + + static Optional getModelDirectory() { + return Optional.ofNullable(modelDirectoryProperty.get()); + } + private void configureModelChoices() { - addRemoteModels(modelChoiceBox); tfModelDirectory.textProperty().bindBidirectional(InstanSegPreferences.modelDirectoryProperty()); handleModelDirectory(tfModelDirectory.getText()); - tfModelDirectory.textProperty().addListener((v, o, n) -> handleModelDirectory(n)); - // for brightfield models, we want to disable the picker and set it to use RGB only + addRemoteModels(modelChoiceBox); + tfModelDirectory.textProperty().addListener((v, o, n) -> { + var oldModelDir = tryToGetPath(o); + if (oldModelDir != null && Files.exists(oldModelDir)) { + watcher.unregister(oldModelDir); + } + handleModelDirectory(n); + addRemoteModels(modelChoiceBox); + }); modelChoiceBox.getSelectionModel().selectedItemProperty().addListener((v, o, n) -> { - if (qupath.getImageData().isBrightfield() && n != null && n.getInputChannels() != InstanSegModel.ANY_CHANNELS) { + if (n == null) { + return; + } + var modelDir = getModelDirectory().orElse(null); + boolean isDownloaded = modelDir != null && n.isDownloaded(modelDir); + if (!isDownloaded || qupath.getImageData() == null) { + return; + } + var numChannels = n.getNumChannels(); + if (qupath.getImageData().isBrightfield() && numChannels.isPresent() && numChannels.get() != InstanSegModel.ANY_CHANNELS) { comboChannels.getCheckModel().clearChecks(); comboChannels.getCheckModel().checkIndices(0, 1, 2); } }); + downloadButton.setOnAction(e -> downloadModel()); + WebView webView = WebViews.create(true); + PopOver infoPopover = new PopOver(webView); + infoButton.setOnAction(e -> { + parseMarkdown(modelChoiceBox.getSelectionModel().getSelectedItem(), webView, infoButton, infoPopover); + }); + } + + private void downloadModel() { + try (var pool = ForkJoinPool.commonPool()) { + pool.execute(() -> { + try { + var modelDir = getModelDirectory().orElse(null); + if (modelDir == null || !Files.exists(modelDir)) { + Dialogs.showErrorMessage(resources.getString("title"), + resources.getString("ui.model-directory.choose-prompt")); + return; + } + var model = modelChoiceBox.getSelectionModel().getSelectedItem(); + Dialogs.showInfoNotification(resources.getString("title"), + String.format(resources.getString("ui.popup.fetching"), model.getName())); + model.download(modelDir); + Dialogs.showInfoNotification(resources.getString("title"), + String.format(resources.getString("ui.popup.available"), model.getName())); + needsUpdating.set(!needsUpdating.get()); + } catch (IOException ex) { + Dialogs.showErrorNotification(resources.getString("title"), resources.getString("error.downloading")); + } + }); + } + } + + private static void parseMarkdown(InstanSegModel model, WebView webView, Button infoButton, PopOver infoPopover) { + Optional readme = model.getREADME(); + if (readme.isEmpty()) return; + String body = readme.get(); + + // Parse the initial markdown only, to extract any YAML front matter + var parser = org.commonmark.parser.Parser.builder().build(); + var doc = parser.parse(body); + + // If the markdown doesn't start with a title, pre-pending the model title & description (if available) + if (!body.startsWith("#")) { + var sb = new StringBuilder(); + sb.append("## ").append(model.getName()).append("\n\n"); + sb.append("----\n\n"); + doc.prependChild(parser.parse(sb.toString())); + } + webView.getEngine().loadContent(HtmlRenderer.builder().build().render(doc)); + infoPopover.show(infoButton); } private static void addRemoteModels(ComboBox comboBox) { - // todo: list models from eg a JSON file + var releases = getReleases(); + if (releases.isEmpty()) { + logger.info("No releases found."); + return; + } + var release = releases.getFirst(); + var assets = getAssets(release); + assets.forEach(asset -> { + comboBox.getItems().add( + InstanSegModel.fromURL( + asset.name.replace(".zip", ""), + asset.browser_download_url) + ); + }); } + private void configureTileSizes() { // The use of 32-bit signed ints for coordinates of the intermediate sparse matrix *might* be // an issue for very large tile sizes - but I haven't seen any evidence of this. @@ -350,12 +519,17 @@ private static void overrideToggleSelected(ToggleButton button) { } private void handleModelDirectory(String n) { - if (n == null) return; - var path = Path.of(n); + var path = tryToGetPath(n); + if (path == null) + return; if (Files.exists(path) && Files.isDirectory(path)) { try { - watcher.register(path); // todo: unregister - addModelsFromPath(n, modelChoiceBox); + var localPath = path.resolve("local"); + if (!Files.exists(localPath)) { + Files.createDirectory(localPath); + } + watcher.register(localPath); // todo: unregister + addModelsFromPath(localPath, modelChoiceBox); } catch (IOException e) { logger.error("Unable to watch directory", e); } @@ -383,7 +557,7 @@ private void addDeviceChoices() { } private void configureMessageLabel() { - messageTextHelper = new MessageTextHelper(modelChoiceBox, deviceChoices, comboChannels); + messageTextHelper = new MessageTextHelper(modelChoiceBox, deviceChoices, comboChannels, needsUpdating); labelMessage.textProperty().bind(messageTextHelper.messageLabelText()); if (messageTextHelper.hasWarning().get()) { labelMessage.getStyleClass().setAll("warning-message"); @@ -398,12 +572,25 @@ private void configureMessageLabel() { }); } - static void addModelsFromPath(String dir, ComboBox box) { - if (dir == null || dir.isEmpty()) return; + private void configureDirectoryLabel() { + isModelDirectoryValid.addListener((v, o, n) -> updateModelDirectoryLabel()); + updateModelDirectoryLabel(); + } + + private void updateModelDirectoryLabel() { + if (isModelDirectoryValid.get()) { + modelDirLabel.getStyleClass().setAll("standard-message"); + modelDirLabel.setText(resources.getString("ui.options.directory")); + } else { + modelDirLabel.getStyleClass().setAll("warning-message"); + modelDirLabel.setText(resources.getString("ui.options.directory-not-set")); + } + } + + static void addModelsFromPath(Path path, ComboBox box) { + if (path == null || !Files.exists(path) || !Files.isDirectory(path)) return; // See https://github.com/controlsfx/controlsfx/issues/1320 box.setItems(FXCollections.observableArrayList()); - var path = Path.of(dir); - if (!Files.exists(path)) return; try (var ps = Files.list(path)) { for (var file: ps.toList()) { if (InstanSegModel.isValidModel(file)) { @@ -416,8 +603,7 @@ static void addModelsFromPath(String dir, ComboBox box) { } void restart() { - executor = Executors.newSingleThreadExecutor(); - executor.submit(watcher::processEvents); + Thread.ofVirtual().start(watcher::processEvents); } @FXML @@ -428,16 +614,46 @@ private void runInstanSeg() { return; } } - - var model = modelChoiceBox.getSelectionModel().getSelectedItem(); ImageServer server = qupath.getImageData().getServer(); - // todo: how to record this in workflow? List selectedChannels = comboChannels .getCheckModel().getCheckedItems() .stream() .filter(Objects::nonNull) .toList(); + + var model = modelChoiceBox.getSelectionModel().getSelectedItem(); + var modelPath = getModelDirectory().orElse(null); + if (modelPath == null) { + Dialogs.showErrorNotification(resources.getString("title"), resources.getString("ui.model-directory.choose-prompt")); + return; + } + if (!model.isDownloaded(modelPath)) { + if (!Dialogs.showYesNoDialog(resources.getString("title"), resources.getString("ui.model-popup"))) + return; + Dialogs.showInfoNotification(resources.getString("title"), String.format(resources.getString("ui.popup.fetching"), model.getName())); + downloadModel(); + if (!model.isDownloaded(modelPath)) { + Dialogs.showErrorNotification(resources.getString("title"), String.format(resources.getString("error.localModel"))); + return; + } + } + + int imageChannels = selectedChannels.size(); + var modelChannels = model.getNumChannels(); + if (modelChannels.isEmpty()) { + Dialogs.showErrorNotification(resources.getString("title"), resources.getString("error.fetching")); + return; + } + + int nModelChannels = modelChannels.get(); + if (nModelChannels != InstanSegModel.ANY_CHANNELS && nModelChannels != imageChannels) { + Dialogs.showErrorNotification(resources.getString("title"), String.format( + resources.getString("ui.error.num-channels-dont-match"), + nModelChannels, imageChannels)); + return; + } + var task = new InstanSegTask(server, model, selectedChannels); pendingTask.set(task); // Reset the pending task when it completes (either successfully or not) @@ -447,6 +663,7 @@ private void runInstanSeg() { pendingTask.set(null); } }); + } private class InstanSegTask extends Task { @@ -474,6 +691,11 @@ protected Void call() { var imageData = qupath.getImageData(); var selectedObjects = imageData.getHierarchy().getSelectionModel().getSelectedObjects(); + Optional path = model.getPath(); + if (path.isEmpty()) { + Dialogs.showErrorNotification(resources.getString("title"), resources.getString("error.querying-local")); + return null; + } var outputChannels = nucleiOnlyCheckBox.isSelected() ? new int[]{0} : new int[]{}; var instanSeg = InstanSeg.builder() @@ -483,7 +705,6 @@ protected Void call() { .outputChannels(outputChannels) .channels(channels.stream().map(ChannelSelectItem::getTransform).toList()) .tileDims(InstanSegPreferences.tileSizeProperty().get()) -// .outputAnnotations() .taskRunner(taskRunner) .build(); @@ -499,7 +720,7 @@ protected Void call() { .build() .%s """, - model.getPath(), + path.get(), deviceChoices.getSelectionModel().getSelectedItem(), outputChannels.length == 0 ? "" : Arrays.stream(outputChannels) .mapToObj(Integer::toString) @@ -515,6 +736,7 @@ protected Void call() { } else { results = instanSeg.detectObjects(selectedObjects); } + imageData.getHierarchy().fireHierarchyChangedEvent(this); imageData.getHistoryWorkflow() .addStep( @@ -550,7 +772,6 @@ private void selectAllTMACores() { hierarchy.getSelectionModel().setSelectedObjects(hierarchy.getTMAGrid().getTMACoreList(), null); } - private void promptToUpdateDirectory(StringProperty dirPath) { var modelDirPath = dirPath.get(); var dir = modelDirPath == null || modelDirPath.isEmpty() ? null : new File(modelDirPath); @@ -569,4 +790,120 @@ else if (!dir.exists()) dirPath.set(newDir.getAbsolutePath()); } + + private static class GitHubRelease { + String tag_name; + String name; + Date published_at; + GitHubAsset[] assets; + String body; + + String getName() { + return name; + } + String getBody() { + return body; + } + Date getDate() { + return published_at; + } + String getTag() { + return tag_name; + } + + @Override + public String toString() { + return name + " with assets:" + Arrays.toString(assets); + } + } + + private static class GitHubAsset { + String name; + String content_type; + URL browser_download_url; + @Override + public String toString() { + return name; + } + + String getType() { + return content_type; + } + + URL getUrl() { + return browser_download_url; + } + + public String getName() { + return name; + } + } + + /** + * Get the list of models from the latest GitHub release, downloading if + * necessary. + * @return A list of GitHub releases, possibly empty. + */ + private static List getReleases() { + Path modelDir = getModelDirectory().orElse(null); + Path cachedReleases = modelDir == null ? null : modelDir.resolve("releases.json"); + + String uString = "https://api.github.com/repos/instanseg/InstanSeg/releases"; + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(uString)) + .GET() + .build(); + HttpResponse response; + String json; + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + // check GitHub api for releases + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + // if response is okay, then cache it + if (response.statusCode() == 200) { + json = response.body(); + if (cachedReleases != null && Files.exists(cachedReleases.getParent())) { + JsonElement jsonElement = JsonParser.parseString(json); + Files.writeString(cachedReleases, gson.toJson(jsonElement)); + } else { + logger.debug("Unable to cache release information - no model directory specified"); + } + } else { + // otherwise problems + throw new IOException("Unable to fetch GitHub release information, status " + response.statusCode()); + } + } catch (IOException | InterruptedException e) { + // if not, try to fall back on a cached version + if (cachedReleases != null && Files.exists(cachedReleases)) { + try { + json = Files.readString(cachedReleases); + } catch (IOException ex) { + logger.warn("Unable to read cached release information"); + return List.of(); + } + } else { + logger.info("Unable to fetch release information from GitHub and no cached version available."); + return List.of(); + } + } + + GitHubRelease[] releases = gson.fromJson(json, GitHubRelease[].class); + if (!(releases.length > 0)) { + logger.info("No releases found in JSON string"); + return List.of(); + } + return List.of(releases); + } + + private static List getAssets(GitHubRelease release) { + var assets = Arrays.stream(release.assets) + .filter(a -> a.getType().equals("application/zip")) + .toList(); + if (assets.isEmpty()) { + logger.info("No valid assets identified for {}", release.name); + } else if (assets.size() > 1) { + logger.info("More than one matching model: {}", release.name); + } + return assets; + } } diff --git a/src/main/java/qupath/ext/instanseg/ui/InstanSegPreferences.java b/src/main/java/qupath/ext/instanseg/ui/InstanSegPreferences.java index c34c590..b98eec0 100644 --- a/src/main/java/qupath/ext/instanseg/ui/InstanSegPreferences.java +++ b/src/main/java/qupath/ext/instanseg/ui/InstanSegPreferences.java @@ -3,6 +3,7 @@ import javafx.beans.property.IntegerProperty; import javafx.beans.property.Property; import javafx.beans.property.StringProperty; +import qupath.lib.common.GeneralTools; import qupath.lib.gui.prefs.PathPrefs; class InstanSegPreferences { @@ -17,7 +18,7 @@ private InstanSegPreferences() { private static final StringProperty preferredDeviceProperty = PathPrefs.createPersistentPreference( "instanseg.pref.device", - "cpu"); + getDefaultDevice()); private static final Property numThreadsProperty = PathPrefs.createPersistentPreference( "instanseg.num.threads", @@ -25,7 +26,20 @@ private InstanSegPreferences() { private static final IntegerProperty tileSizeProperty = PathPrefs.createPersistentPreference( "intanseg.tile.size", - 256); + 512); + + /** + * MPS should work reliably (and much faster) on Apple Silicon, so set as default. + * Everywhere else, use CPU as we can't count on a GPU/CUDA being available. + * @return + */ + private static String getDefaultDevice() { + if (GeneralTools.isMac() && "aarch64".equals(System.getProperty("os.arch"))) { + return "mps"; + } else { + return "cpu"; + } + } static StringProperty modelDirectoryProperty() { return modelDirectoryProperty; diff --git a/src/main/java/qupath/ext/instanseg/ui/MessageTextHelper.java b/src/main/java/qupath/ext/instanseg/ui/MessageTextHelper.java index 8f869b5..af95f1e 100644 --- a/src/main/java/qupath/ext/instanseg/ui/MessageTextHelper.java +++ b/src/main/java/qupath/ext/instanseg/ui/MessageTextHelper.java @@ -5,6 +5,7 @@ import javafx.beans.binding.BooleanBinding; import javafx.beans.binding.ObjectBinding; import javafx.beans.binding.StringBinding; +import javafx.beans.property.BooleanProperty; import javafx.beans.property.IntegerProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleIntegerProperty; @@ -28,12 +29,15 @@ * Helper class for determining which text to display in the message label. */ class MessageTextHelper { + private static final ResourceBundle resources = ResourceBundle.getBundle("qupath.ext.instanseg.ui.strings"); - private static final QuPathGUI qupath = QuPathGUI.getInstance(); + + private final QuPathGUI qupath = QuPathGUI.getInstance(); private final SelectedObjectCounter selectedObjectCounter; private final SearchableComboBox modelChoiceBox; private final ChoiceBox deviceChoiceBox; private final CheckComboBox comboChannels; + private final BooleanProperty needsUpdating; /** * Text to display a warning (because inference can't be run) @@ -55,10 +59,11 @@ class MessageTextHelper { */ private BooleanBinding hasWarning; - MessageTextHelper(SearchableComboBox modelChoiceBox, ChoiceBox deviceChoiceBox, CheckComboBox comboChannels) { + MessageTextHelper(SearchableComboBox modelChoiceBox, ChoiceBox deviceChoiceBox, CheckComboBox comboChannels, BooleanProperty needsUpdating) { this.modelChoiceBox = modelChoiceBox; this.deviceChoiceBox = deviceChoiceBox; this.comboChannels = comboChannels; + this.needsUpdating = needsUpdating; this.selectedObjectCounter = new SelectedObjectCounter(qupath.imageDataProperty()); configureMessageTextBindings(); } @@ -110,7 +115,8 @@ private StringBinding createWarningTextBinding() { comboChannels.getCheckModel().getCheckedItems(), deviceChoiceBox.getSelectionModel().selectedItemProperty(), selectedObjectCounter.numSelectedAnnotations, - selectedObjectCounter.numSelectedTMACores); + selectedObjectCounter.numSelectedTMACores, + needsUpdating); } private String getWarningText() { @@ -123,14 +129,21 @@ private String getWarningText() { return resources.getString("ui.error.no-selection"); if (deviceChoiceBox.getSelectionModel().isEmpty()) return resources.getString("ui.error.no-device"); - int modelChannels = modelChoiceBox.getSelectionModel().getSelectedItem().getInputChannels(); - int selectedChannels = comboChannels.getCheckModel().getCheckedItems().size(); - if (modelChannels != InstanSegModel.ANY_CHANNELS) { - if (modelChannels != selectedChannels) { - return String.format( - resources.getString("ui.error.num-channels-dont-match"), - modelChannels, - selectedChannels); + var modelDir = InstanSegController.getModelDirectory().orElse(null); + if (modelDir != null && modelChoiceBox.getSelectionModel().getSelectedItem().isDownloaded(modelDir)) { + // shouldn't happen if downloaded anyway! + var modelChannels = modelChoiceBox.getSelectionModel().getSelectedItem().getNumChannels(); + if (modelChannels.isPresent()) { + int nModelChannels = modelChannels.get(); + int selectedChannels = comboChannels.getCheckModel().getCheckedItems().size(); + if (nModelChannels != InstanSegModel.ANY_CHANNELS) { + if (nModelChannels != selectedChannels) { + return String.format( + resources.getString("ui.error.num-channels-dont-match"), + nModelChannels, + selectedChannels); + } + } } } return null; diff --git a/src/main/java/qupath/ext/instanseg/ui/Watcher.java b/src/main/java/qupath/ext/instanseg/ui/Watcher.java index cea0d8f..4ef2de0 100644 --- a/src/main/java/qupath/ext/instanseg/ui/Watcher.java +++ b/src/main/java/qupath/ext/instanseg/ui/Watcher.java @@ -1,6 +1,7 @@ package qupath.ext.instanseg.ui; import javafx.application.Platform; +import javafx.collections.FXCollections; import org.controlsfx.control.SearchableComboBox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,7 @@ void register(Path dir) throws IOException { keys.put(key, dir); } - private void unregister(Path dir) { + void unregister(Path dir) { for (var es: keys.entrySet()) { if (es.getValue().equals(dir)) { logger.debug("Unregister: {}", es.getValue()); @@ -79,27 +80,28 @@ void processEvents() { for (WatchEvent event : key.pollEvents()) { WatchEvent.Kind kind = event.kind(); WatchEvent ev = cast(event); - Path name = ev.context(); + Path fileName = ev.context(); // Context for directory entry event is the file name of entry - Path child = dir.resolve(name); + Path filePath = dir.resolve(fileName); // print out event - logger.debug("{}: {}", event.kind().name(), child); - - if (kind == ENTRY_CREATE && InstanSegModel.isValidModel(name)) { - Platform.runLater(() -> { - try { - modelChoiceBox.getItems().add(InstanSegModel.fromPath(child)); - } catch (IOException e) { - logger.error("Unable to add model from path", e); - } - }); + logger.debug("{}: {}", event.kind().name(), filePath); + + if (kind == ENTRY_CREATE) { + if (InstanSegModel.isValidModel(filePath)) { + Platform.runLater(() -> { + try { + modelChoiceBox.getItems().add(InstanSegModel.fromPath(filePath)); + } catch (IOException e) { + logger.error("Unable to add model from path", e); + } + }); + } } - if (kind == ENTRY_DELETE && InstanSegModel.isValidModel(name)) { - Platform.runLater(() -> { - modelChoiceBox.getItems().removeIf(model -> model.getPath().equals(child)); - }); + if (kind == ENTRY_DELETE) { + // todo: controller should handle adding/removing logic + removeModel(filePath); } } @@ -120,4 +122,12 @@ void processEvents() { void interrupt() { interrupted = true; } + + private void removeModel(Path filePath) { + // https://github.com/controlsfx/controlsfx/issues/1320 + var items = FXCollections.observableArrayList(modelChoiceBox.getItems()); + var matchingItems = modelChoiceBox.getItems().stream().filter(model -> model.getPath().map(p -> p.equals(filePath)).orElse(false)).toList(); + items.removeAll(matchingItems); + Platform.runLater(() -> modelChoiceBox.setItems(items)); + } } diff --git a/src/main/resources/qupath/ext/instanseg/ui/instanseg_control.fxml b/src/main/resources/qupath/ext/instanseg/ui/instanseg_control.fxml index cd54945..05f84f0 100644 --- a/src/main/resources/qupath/ext/instanseg/ui/instanseg_control.fxml +++ b/src/main/resources/qupath/ext/instanseg/ui/instanseg_control.fxml @@ -1,16 +1,32 @@ - - - - - - + - + + + + + + + + + + + + + + + + + + + + + +
- + @@ -91,11 +107,11 @@ - + - + -