Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement model versioning #113

Merged
merged 20 commits into from
Dec 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/java/qupath/ext/instanseg/core/InstanSeg.java
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ private InstanSegResults runInstanSeg(ImageData<BufferedImage> imageData, Collec
.postProcess(postProcessor)
.downsample(downsample)
.build();

processor.processObjects(taskRunner, imageData, pathObjects);
int nObjects = pathObjects.stream().mapToInt(PathObject::nChildObjects).sum();
if (predictionProcessor instanceof TilePredictionProcessor tileProcessor) {
Expand Down
81 changes: 48 additions & 33 deletions src/main/java/qupath/ext/instanseg/core/InstanSegModel.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
public class InstanSegModel {

private static final Logger logger = LoggerFactory.getLogger(InstanSegModel.class);
private String version;
private URL modelURL = null;

/**
Expand All @@ -44,11 +45,13 @@ public class InstanSegModel {
private InstanSegModel(BioimageIoSpec.BioimageIoModel bioimageIoModel) {
this.model = bioimageIoModel;
this.path = Paths.get(model.getBaseURI());
this.version = model.getVersion();
this.name = model.getName();
}

private InstanSegModel(String name, URL modelURL) {
private InstanSegModel(String name, String version, URL modelURL) {
this.name = name;
this.version = version;
this.modelURL = modelURL;
}

Expand All @@ -68,46 +71,38 @@ public static InstanSegModel fromPath(Path path) throws IOException {
* @param browserDownloadUrl The download URL from eg GitHub
* @return A handle on the created model
*/
public static InstanSegModel fromURL(String name, URL browserDownloadUrl) {
return new InstanSegModel(name, browserDownloadUrl);
public static InstanSegModel fromURL(String name, String version, URL browserDownloadUrl) {
return new InstanSegModel(name, version, browserDownloadUrl);
}

/**
* Check if the model has been downloaded already.
* @return True if a flag has been set.
* @return True if the model has a known path that exists and is valid, or if a suitable directory can be found in the localModelPath
*/
public boolean isDownloaded(Path localModelPath) {
public boolean isValid() {
// Check path first - *sometimes* the model might be downloaded, but have a name
// that doesn't match with the filename (although we'd prefer this didn't happen...)
if (path != null && model != null && Files.exists(path))
if (path != null && model != null && isValidModel(path))
return true;
// todo: this should also check if the contents are what we expect
if (Files.exists(localModelPath.resolve(name))) {
try {
download(localModelPath);
} catch (IOException e) {
logger.error("Model directory exists but is not valid", e);
}
} else {
// The model may have been deleted or renamed - we won't be able to load it
return false;
}
return path != null && model != null;
// The model may have been deleted or renamed - we won't be able to load it
return false;
}

/**
* Trigger a download for a model
* @throws IOException If an error occurs when downloading, unzipping, etc.
*/
public void download(Path localModelPath) throws IOException {
if (path != null && Files.exists(path) && model != null)
public void download(Path downloadedModelDir) throws IOException {
if (path != null && isValidModel(path) && model != null) {
return;
}
var zipFile = downloadZipIfNeeded(
this.modelURL,
localModelPath,
name);
downloadedModelDir,
getFolderName(name, version));
this.path = unzipIfNeeded(zipFile);
this.model = BioimageIoSpec.parseModel(path.toFile());
this.version = model.getVersion();
}

/**
Expand Down Expand Up @@ -213,8 +208,8 @@ public Optional<Path> getPath() {
@Override
public String toString() {
String name = getName();
String parent = getPath().map(Path::getFileName).map(Path::toString).orElse(null);
String version = getModel().map(BioimageIoSpec.BioimageIoModel::getVersion).orElse(null);
String parent = getPath().map(Path::getParent).map(Path::getFileName).map(Path::toString).orElse(null);
String version = getModel().map(BioimageIoSpec.BioimageIoModel::getVersion).orElse(this.version);
if (parent != null && !parent.equals(name)) {
name = parent + "/" + name;
}
Expand Down Expand Up @@ -271,8 +266,9 @@ private Optional<BioimageIoSpec.BioimageIoModel> getModel() {
return Optional.ofNullable(model);
}

private static Path downloadZipIfNeeded(URL url, Path localDirectory, String filename) throws IOException {
var zipFile = localDirectory.resolve(Path.of(filename + ".zip"));
private static Path downloadZipIfNeeded(URL url, Path downloadDirectory, String filename) throws IOException {
Files.createDirectories(downloadDirectory);
var zipFile = downloadDirectory.resolve(filename + ".zip");
if (!isDownloadedAlready(zipFile)) {
try (InputStream stream = url.openStream()) {
try (ReadableByteChannel readableByteChannel = Channels.newChannel(stream)) {
Expand All @@ -286,32 +282,51 @@ private static Path downloadZipIfNeeded(URL url, Path localDirectory, String fil
}

private static boolean isDownloadedAlready(Path zipFile) {
// todo: validate contents somehow
return Files.exists(zipFile);
if (!Files.exists(zipFile)) {
return false;
}
try {
BioimageIoSpec.parseModel(zipFile.toFile());
} catch (IOException e) {
logger.warn("Invalid zip file", e);
return false;
}
return true;
}

private static Path unzipIfNeeded(Path zipFile) throws IOException {
var outdir = zipFile.resolveSibling(zipFile.getFileName().toString().replace(".zip", ""));
private Path unzipIfNeeded(Path zipFile) throws IOException {
var zipSpec = BioimageIoSpec.parseModel(zipFile);
String version = zipSpec.getVersion();
var outdir = zipFile.resolveSibling(getFolderName(zipSpec.getName(), version));
if (!isUnpackedAlready(outdir)) {
try {
unzip(zipFile, zipFile.getParent());
unzip(zipFile, outdir);
// Files.delete(zipFile);
} catch (IOException e) {
logger.error("Error unzipping model", e);
// clean up files just in case!
Files.deleteIfExists(zipFile);
Files.deleteIfExists(outdir);
} finally {
Files.deleteIfExists(zipFile);
}
}
return outdir;
}

private String getFolderName(String name, String version) {
if (version == null) {
return name;
}
return name + "-" + version;
}

private static boolean isUnpackedAlready(Path outdir) {
return Files.exists(outdir) && isValidModel(outdir);
}

private static void unzip(Path zipFile, Path destination) throws IOException {
if (!Files.exists(destination)) {
Files.createDirectory(destination);
Files.createDirectories(destination);
}
ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile.toFile())));
ZipEntry entry = zipIn.getNextEntry();
Expand Down
147 changes: 0 additions & 147 deletions src/main/java/qupath/ext/instanseg/ui/GitHubUtils.java

This file was deleted.

Loading
Loading