diff --git a/src/main/java/qupath/ext/instanseg/core/InstanSeg.java b/src/main/java/qupath/ext/instanseg/core/InstanSeg.java index b087961..1238cb3 100644 --- a/src/main/java/qupath/ext/instanseg/core/InstanSeg.java +++ b/src/main/java/qupath/ext/instanseg/core/InstanSeg.java @@ -247,6 +247,7 @@ private InstanSegResults runInstanSeg(ImageData imageData, Collec .postProcess(postProcessor) .downsample(downsample) .build(); + processor.processObjects(taskRunner, imageData, pathObjects); int nObjects = pathObjects.stream().mapToInt(PathObject::nChildObjects).sum(); if (predictionProcessor instanceof TilePredictionProcessor tileProcessor) { diff --git a/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java b/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java index 5c9a36c..3b35262 100644 --- a/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java +++ b/src/main/java/qupath/ext/instanseg/core/InstanSegModel.java @@ -30,6 +30,7 @@ public class InstanSegModel { private static final Logger logger = LoggerFactory.getLogger(InstanSegModel.class); + private String version; private URL modelURL = null; /** @@ -44,11 +45,13 @@ public class InstanSegModel { private InstanSegModel(BioimageIoSpec.BioimageIoModel bioimageIoModel) { this.model = bioimageIoModel; this.path = Paths.get(model.getBaseURI()); + this.version = model.getVersion(); this.name = model.getName(); } - private InstanSegModel(String name, URL modelURL) { + private InstanSegModel(String name, String version, URL modelURL) { this.name = name; + this.version = version; this.modelURL = modelURL; } @@ -68,46 +71,38 @@ public static InstanSegModel fromPath(Path path) throws IOException { * @param browserDownloadUrl The download URL from eg GitHub * @return A handle on the created model */ - public static InstanSegModel fromURL(String name, URL browserDownloadUrl) { - return new InstanSegModel(name, browserDownloadUrl); + public static InstanSegModel fromURL(String name, String version, URL browserDownloadUrl) { + return new InstanSegModel(name, version, browserDownloadUrl); } /** * Check if the model has been downloaded already. - * @return True if a flag has been set. + * @return True if the model has a known path that exists and is valid, or if a suitable directory can be found in the localModelPath */ - public boolean isDownloaded(Path localModelPath) { + public boolean isValid() { // Check path first - *sometimes* the model might be downloaded, but have a name // that doesn't match with the filename (although we'd prefer this didn't happen...) - if (path != null && model != null && Files.exists(path)) + if (path != null && model != null && isValidModel(path)) return true; - // todo: this should also check if the contents are what we expect - if (Files.exists(localModelPath.resolve(name))) { - try { - download(localModelPath); - } catch (IOException e) { - logger.error("Model directory exists but is not valid", e); - } - } else { - // The model may have been deleted or renamed - we won't be able to load it - return false; - } - return path != null && model != null; + // The model may have been deleted or renamed - we won't be able to load it + return false; } /** * Trigger a download for a model * @throws IOException If an error occurs when downloading, unzipping, etc. */ - public void download(Path localModelPath) throws IOException { - if (path != null && Files.exists(path) && model != null) + public void download(Path downloadedModelDir) throws IOException { + if (path != null && isValidModel(path) && model != null) { return; + } var zipFile = downloadZipIfNeeded( this.modelURL, - localModelPath, - name); + downloadedModelDir, + getFolderName(name, version)); this.path = unzipIfNeeded(zipFile); this.model = BioimageIoSpec.parseModel(path.toFile()); + this.version = model.getVersion(); } /** @@ -213,8 +208,8 @@ public Optional getPath() { @Override public String toString() { String name = getName(); - String parent = getPath().map(Path::getFileName).map(Path::toString).orElse(null); - String version = getModel().map(BioimageIoSpec.BioimageIoModel::getVersion).orElse(null); + String parent = getPath().map(Path::getParent).map(Path::getFileName).map(Path::toString).orElse(null); + String version = getModel().map(BioimageIoSpec.BioimageIoModel::getVersion).orElse(this.version); if (parent != null && !parent.equals(name)) { name = parent + "/" + name; } @@ -271,8 +266,9 @@ private Optional getModel() { return Optional.ofNullable(model); } - private static Path downloadZipIfNeeded(URL url, Path localDirectory, String filename) throws IOException { - var zipFile = localDirectory.resolve(Path.of(filename + ".zip")); + private static Path downloadZipIfNeeded(URL url, Path downloadDirectory, String filename) throws IOException { + Files.createDirectories(downloadDirectory); + var zipFile = downloadDirectory.resolve(filename + ".zip"); if (!isDownloadedAlready(zipFile)) { try (InputStream stream = url.openStream()) { try (ReadableByteChannel readableByteChannel = Channels.newChannel(stream)) { @@ -286,32 +282,51 @@ private static Path downloadZipIfNeeded(URL url, Path localDirectory, String fil } private static boolean isDownloadedAlready(Path zipFile) { - // todo: validate contents somehow - return Files.exists(zipFile); + if (!Files.exists(zipFile)) { + return false; + } + try { + BioimageIoSpec.parseModel(zipFile.toFile()); + } catch (IOException e) { + logger.warn("Invalid zip file", e); + return false; + } + return true; } - private static Path unzipIfNeeded(Path zipFile) throws IOException { - var outdir = zipFile.resolveSibling(zipFile.getFileName().toString().replace(".zip", "")); + private Path unzipIfNeeded(Path zipFile) throws IOException { + var zipSpec = BioimageIoSpec.parseModel(zipFile); + String version = zipSpec.getVersion(); + var outdir = zipFile.resolveSibling(getFolderName(zipSpec.getName(), version)); if (!isUnpackedAlready(outdir)) { try { - unzip(zipFile, zipFile.getParent()); + unzip(zipFile, outdir); // Files.delete(zipFile); } catch (IOException e) { + logger.error("Error unzipping model", e); // clean up files just in case! - Files.deleteIfExists(zipFile); Files.deleteIfExists(outdir); + } finally { + Files.deleteIfExists(zipFile); } } return outdir; } + private String getFolderName(String name, String version) { + if (version == null) { + return name; + } + return name + "-" + version; + } + private static boolean isUnpackedAlready(Path outdir) { return Files.exists(outdir) && isValidModel(outdir); } private static void unzip(Path zipFile, Path destination) throws IOException { if (!Files.exists(destination)) { - Files.createDirectory(destination); + Files.createDirectories(destination); } ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile.toFile()))); ZipEntry entry = zipIn.getNextEntry(); diff --git a/src/main/java/qupath/ext/instanseg/ui/GitHubUtils.java b/src/main/java/qupath/ext/instanseg/ui/GitHubUtils.java deleted file mode 100644 index 31aee16..0000000 --- a/src/main/java/qupath/ext/instanseg/ui/GitHubUtils.java +++ /dev/null @@ -1,147 +0,0 @@ -package qupath.ext.instanseg.ui; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Date; -import java.util.List; - -public class GitHubUtils { - - private static final Logger logger = LoggerFactory.getLogger(GitHubUtils.class); - - static class GitHubRelease { - - private String tag_name; - private String name; - private Date published_at; - private GitHubAsset[] assets; - private String body; - - String getName() { - return name; - } - String getBody() { - return body; - } - Date getDate() { - return published_at; - } - String getTag() { - return tag_name; - } - - @Override - public String toString() { - return name + " with assets:" + Arrays.toString(assets); - } - } - - static class GitHubAsset { - - private String name; - private String content_type; - private URL browser_download_url; - - String getType() { - return content_type; - } - - URL getUrl() { - return browser_download_url; - } - - String getName() { - return name; - } - - @Override - public String toString() { - return name; - } - - } - - /** - * Get the list of models from the latest GitHub release, downloading if - * necessary. - * @return A list of GitHub releases, possibly empty. - */ - static List getReleases(Path modelDir) { - Path cachedReleases = modelDir == null ? null : modelDir.resolve("releases.json"); - - String uString = "https://api.github.com/repos/instanseg/InstanSeg/releases"; - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(uString)) - .GET() - .build(); - HttpResponse response; - String json; - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - // check GitHub api for releases - try (HttpClient client = HttpClient.newHttpClient()) { - response = client.send(request, HttpResponse.BodyHandlers.ofString()); - // if response is okay, then cache it - if (response.statusCode() == 200) { - json = response.body(); - if (cachedReleases != null && Files.exists(cachedReleases.getParent())) { - JsonElement jsonElement = JsonParser.parseString(json); - Files.writeString(cachedReleases, gson.toJson(jsonElement)); - } else { - logger.debug("Unable to cache release information - no model directory specified"); - } - } else { - // otherwise problems - throw new IOException("Unable to fetch GitHub release information, status " + response.statusCode()); - } - } catch (IOException | InterruptedException e) { - // if not, try to fall back on a cached version - if (cachedReleases != null && Files.exists(cachedReleases)) { - try { - json = Files.readString(cachedReleases); - } catch (IOException ex) { - logger.warn("Unable to read cached release information"); - return List.of(); - } - } else { - logger.info("Unable to fetch release information from GitHub and no cached version available."); - return List.of(); - } - } - - GitHubRelease[] releases = gson.fromJson(json, GitHubRelease[].class); - if (!(releases.length > 0)) { - logger.info("No releases found in JSON string"); - return List.of(); - } - return List.of(releases); - } - - - static List getAssets(GitHubRelease release) { - var assets = Arrays.stream(release.assets) - .filter(a -> a.getType().equals("application/zip")) - .toList(); - if (assets.isEmpty()) { - logger.info("No valid assets identified for {}", release.name); - } else if (assets.size() > 1) { - logger.info("More than one matching model: {}", release.name); - } - return assets; - } - - -} diff --git a/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java b/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java index ad91d8d..09220ca 100644 --- a/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java +++ b/src/main/java/qupath/ext/instanseg/ui/InstanSegController.java @@ -1,5 +1,6 @@ package qupath.ext.instanseg.ui; +import com.google.gson.Gson; import javafx.application.Platform; import javafx.beans.binding.Bindings; import javafx.beans.binding.BooleanBinding; @@ -50,8 +51,11 @@ import java.awt.image.BufferedImage; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -68,6 +72,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.FutureTask; +import java.util.stream.Collectors; import java.util.stream.IntStream; /** @@ -211,7 +216,6 @@ private void refreshAvailableModels() { } } var remoteAndNotLocal = remoteModels.stream() - .filter(m -> !localModelNames.containsKey(m.getName())) .sorted(comparator) .toList(); list.addAll(localModels); @@ -369,8 +373,7 @@ private void updateInputChannels(ImageData imageData) { // if brightfield, then check R, G, and B comboInputChannels.getCheckModel().checkIndices(IntStream.range(0, 3).toArray()); var modelDir = InstanSegUtils.getModelDirectory().orElse(null); - // todo: not clear why this is needed. is this handling the checkcombobox weirdness on clearing checks, or? - if (model != null && modelDir != null && model.isDownloaded(modelDir)) { + if (model != null && modelDir != null && model.isValid()) { var modelChannels = model.getNumChannels(); if (modelChannels.isPresent()) { int nModelChannels = modelChannels.get(); @@ -459,7 +462,7 @@ private void configureRunning() { return true; } var modelDir = InstanSegUtils.getModelDirectory().orElse(null); - if (modelDir != null && !model.isDownloaded(modelDir)) { + if (modelDir != null && !model.isValid()) { return false; // to enable "download and run" } return false; @@ -483,7 +486,7 @@ private void refreshModelChoice() { return; var modelDir = InstanSegUtils.getModelDirectory().orElse(null); - boolean isDownloaded = modelDir != null && model.isDownloaded(modelDir); + boolean isDownloaded = modelDir != null && model.isValid(); if (!isDownloaded || qupath.getImageData() == null) { return; } @@ -540,7 +543,7 @@ private InstanSegModel downloadModel(InstanSegModel model, Path modelDir) { try { Dialogs.showInfoNotification(resources.getString("title"), String.format(resources.getString("ui.popup.fetching"), model.getName())); - model.download(modelDir); + model.download(modelDir.resolve("downloaded")); Dialogs.showInfoNotification(resources.getString("title"), String.format(resources.getString("ui.popup.available"), model.getName())); FXUtils.runOnApplicationThread(() -> { @@ -549,6 +552,7 @@ private InstanSegModel downloadModel(InstanSegModel model, Path modelDir) { }); } catch (IOException ex) { Dialogs.showErrorNotification(resources.getString("title"), resources.getString("error.downloading")); + logger.error("Error download model", ex); } return model; } @@ -613,19 +617,21 @@ private static List getRemoteModels() { return List.of(); } } - var releases = GitHubUtils.getReleases(InstanSegUtils.getModelDirectory().orElse(null)); - if (releases.isEmpty()) { - logger.info("No releases found."); - return List.of(); - } - var release = releases.getFirst(); - var assets = GitHubUtils.getAssets(release); + InputStream in = InstanSegController.class.getResourceAsStream("model-index.json"); + String cont = new BufferedReader(new InputStreamReader(in)) + .lines() + .collect(Collectors.joining("\n")); + var gson = new Gson(); + var remoteModels = gson.fromJson(cont, RemoteModel[].class); + List models = new ArrayList<>(); - for (var asset : assets) { + for (var remoteModel: remoteModels) { models.add( InstanSegModel.fromURL( - asset.getName().replace(".zip", ""), - asset.getUrl()) + remoteModel.getName(), + remoteModel.getVersion(), + remoteModel.getUrl() + ) ); } return List.copyOf(models); @@ -759,12 +765,12 @@ private void runInstanSeg(InstanSegModel model) { return; } - if (!model.isDownloaded(modelPath)) { + if (!model.isValid()) { if (!Dialogs.showYesNoDialog(resources.getString("title"), resources.getString("ui.model-popup"))) return; downloadModelAsync(model) .thenAccept((InstanSegModel suppliedModel) -> { - if (suppliedModel == null || !suppliedModel.isDownloaded(modelPath)) { + if (suppliedModel == null || !suppliedModel.isValid()) { Dialogs.showErrorNotification(resources.getString("title"), String.format(resources.getString("error.localModel"))); } else { runInstanSeg(suppliedModel); diff --git a/src/main/java/qupath/ext/instanseg/ui/InstanSegUtils.java b/src/main/java/qupath/ext/instanseg/ui/InstanSegUtils.java index 2535b5d..e057abd 100644 --- a/src/main/java/qupath/ext/instanseg/ui/InstanSegUtils.java +++ b/src/main/java/qupath/ext/instanseg/ui/InstanSegUtils.java @@ -70,7 +70,7 @@ public static BooleanBinding createModelDownloadedBinding(ObservableValue modelDirectoryBinding = InstanSegUtils.getModelDirectoryBinding(); - private static Watcher instance = new Watcher(); + private static final Watcher instance = new Watcher(); private Watcher() { modelDirectoryBinding.addListener(this::handleModelDirectoryChange); @@ -74,11 +74,11 @@ private void handleModelDirectoryChange(ObservableValue observab // Currently, we look *only* in the model directory for models // But we could register subdirectories here if we wanted (e.g. 'local', 'downloaded') if (oldPath != null) { - unregister(oldPath); + unregister(oldPath.resolve("local")); } - if (newPath != null && Files.isDirectory(newPath)) { + if (newPath != null && Files.isDirectory(newPath.resolve("local"))) { try { - register(newPath); + register(newPath.resolve("local")); } catch (IOException e) { logger.error("Unable to register new model directory", e); } @@ -130,6 +130,7 @@ private void processEvents() { if (key.watchable() instanceof Path dir) { for (WatchEvent rawEvent : key.pollEvents()) { + @SuppressWarnings("unchecked") WatchEvent event = (WatchEvent)rawEvent; WatchEvent.Kind kind = event.kind(); Path fileName = event.context(); diff --git a/src/main/resources/qupath/ext/instanseg/ui/model-index.json b/src/main/resources/qupath/ext/instanseg/ui/model-index.json new file mode 100644 index 0000000..bcb3547 --- /dev/null +++ b/src/main/resources/qupath/ext/instanseg/ui/model-index.json @@ -0,0 +1,14 @@ +[ + { + "name": "brightfield_nuclei", + "url": "https://github.com/alanocallaghan/instanseg/releases/download/v0.1.0/brightfield_nuclei.zip", + "version": "0.1.0", + "license": "Apache-2.0" + }, + { + "name": "fluorescence_nuclei_and_cells", + "url": "https://github.com/alanocallaghan/instanseg/releases/download/v0.1.0/fluorescence_nuclei_and_cells.zip", + "version": "0.1.0", + "license": "Apache-2.0" + } +]