From 748e99806b3be06d5c7ac06a538698f7c523cb26 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Sun, 2 Feb 2025 12:48:52 -0500 Subject: [PATCH] add support to load alignment pipeline parameters from URL-based resources to facilitate usage in Google Cloud Spark runs --- .../alignment/util/UrlResourceUtil.java | 53 ++++++ .../parameter/MultiProjectParameters.java | 4 + .../pipeline/AlignmentPipelineClient.java | 30 +++- .../pipeline/AlignmentPipelineParameters.java | 23 ++- .../01_match/pipe.01.match-patch-check.json | 156 +++++++++++++++++ .../pipeline_json/01_match/pipe.01.match.json | 143 +++++++++++++++ .../pipe.01.mip-match-patch-check.json | 164 ++++++++++++++++++ .../AlignmentPipelineParametersTest.java | 3 +- 8 files changed, 570 insertions(+), 6 deletions(-) create mode 100644 render-app/src/main/java/org/janelia/alignment/util/UrlResourceUtil.java create mode 100644 render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match-patch-check.json create mode 100644 render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match.json create mode 100644 render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.mip-match-patch-check.json diff --git a/render-app/src/main/java/org/janelia/alignment/util/UrlResourceUtil.java b/render-app/src/main/java/org/janelia/alignment/util/UrlResourceUtil.java new file mode 100644 index 000000000..12f737841 --- /dev/null +++ b/render-app/src/main/java/org/janelia/alignment/util/UrlResourceUtil.java @@ -0,0 +1,53 @@ +package org.janelia.alignment.util; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.net.URLConnection; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipInputStream; + +/** + * Shared URL management utilities. + * + * @author Eric Trautman + */ +public class UrlResourceUtil { + + public static final UrlResourceUtil DEFAULT_INSTANCE = new UrlResourceUtil(); + + private final int bufferSize; + + public UrlResourceUtil() { + this(DEFAULT_BUFFER_SIZE); + } + + public UrlResourceUtil(final int bufferSize) { + this.bufferSize = bufferSize; + } + + public Reader getExtensionBasedReader(final URL url) + throws IOException { + + final InputStream inputStream; + + final URLConnection connection = url.openConnection(); + + final String path = url.getPath(); + if (path.endsWith(".gz")) { + inputStream = new GZIPInputStream(connection.getInputStream()); + } else if (path.endsWith(".zip")) { + inputStream = new ZipInputStream(connection.getInputStream()); + } else { + inputStream = new BufferedInputStream(connection.getInputStream(), bufferSize); + } + + return new InputStreamReader(inputStream); + } + + private static final int DEFAULT_BUFFER_SIZE = 65536; + +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java index a3396bbc1..8c9f2e704 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java @@ -82,6 +82,10 @@ public String getBaseDataUrl() { return baseDataUrl; } + public void setBaseDataUrl(final String baseDataUrl) { + this.baseDataUrl = baseDataUrl; + } + public MatchCollectionId getMatchCollectionIdForStack(final StackId stackId) { return matchCollection == null ? stackId.getDefaultMatchCollectionId(deriveMatchCollectionNamesFromProject) : diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineClient.java index 7d4d8af0d..987e1f54d 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineClient.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.Serializable; +import java.net.URL; import java.util.List; import org.apache.spark.SparkConf; @@ -25,9 +26,14 @@ public class AlignmentPipelineClient public static class Parameters extends CommandLineParameters { @Parameter( names = "--pipelineJson", - description = "JSON file where pipeline parameters are defined", + description = "JSON file (or URL) where pipeline parameters are defined", required = true) public String pipelineJson; + + @Parameter( + names = "--baseDataUrl", + description = "Override the baseDataUrl defined in the pipeline JSON (omit to use JSON value)") + public String baseDataUrl; } /** Run the client with command line parameters. */ @@ -51,10 +57,28 @@ private AlignmentPipelineClient() { public void createContextAndRun(final Parameters clientParameters) throws IOException, IllegalArgumentException { final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + LOG.info("createContextAndRun: appId is {}", sparkContext.getConf().getAppId()); - final AlignmentPipelineParameters parsedParameters = - AlignmentPipelineParameters.fromJsonFile(clientParameters.pipelineJson); + + if (clientParameters.pipelineJson == null) { + throw new IllegalArgumentException("pipelineJson is required"); + } + + final AlignmentPipelineParameters parsedParameters; + if (clientParameters.pipelineJson.startsWith("/") || + clientParameters.pipelineJson.startsWith("\\") || + clientParameters.pipelineJson.startsWith("file:")) { + parsedParameters = + AlignmentPipelineParameters.fromJsonFile(clientParameters.pipelineJson, + clientParameters.baseDataUrl); + } else { + final URL jsonUrl = new URL(clientParameters.pipelineJson); + parsedParameters = AlignmentPipelineParameters.fromJsonUrl(jsonUrl, + clientParameters.baseDataUrl); + } + runPipeline(sparkContext, parsedParameters); + } } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index 81d567f54..699ff60a1 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.Reader; import java.io.Serializable; +import java.net.URL; import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.List; @@ -12,6 +13,7 @@ import org.janelia.alignment.spec.stack.PipelineStackIdNamingGroups; import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.alignment.util.FileUtil; +import org.janelia.alignment.util.UrlResourceUtil; import org.janelia.render.client.newsolver.setup.AffineBlockSolverSetup; import org.janelia.render.client.newsolver.setup.IntensityCorrectionSetup; import org.janelia.render.client.parameter.MFOVMontageMatchPatchParameters; @@ -195,12 +197,29 @@ public static AlignmentPipelineParameters fromJson(final Reader json) { return STRICT_JSON_HELPER.fromJson(json); } - public static AlignmentPipelineParameters fromJsonFile(final String dataFile) + public static AlignmentPipelineParameters fromJsonFile(final String jsonFilePathString, + final String baseDataUrlString) throws IOException { final AlignmentPipelineParameters parameters; - final Path path = FileSystems.getDefault().getPath(dataFile).toAbsolutePath(); + final Path path = FileSystems.getDefault().getPath(jsonFilePathString).toAbsolutePath(); try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) { parameters = fromJson(reader); + if (baseDataUrlString != null) { + parameters.multiProject.setBaseDataUrl(baseDataUrlString); + } + } + return parameters; + } + + public static AlignmentPipelineParameters fromJsonUrl(final URL jsonUrl, + final String baseDataUrlString) + throws IOException { + final AlignmentPipelineParameters parameters; + try (final Reader reader = UrlResourceUtil.DEFAULT_INSTANCE.getExtensionBasedReader(jsonUrl)) { + parameters = fromJson(reader); + if (baseDataUrlString != null) { + parameters.multiProject.setBaseDataUrl(baseDataUrlString); + } } return parameters; } diff --git a/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match-patch-check.json b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match-patch-check.json new file mode 100644 index 000000000..81d04e316 --- /dev/null +++ b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match-patch-check.json @@ -0,0 +1,156 @@ +{ + "multiProject": { + "baseDataUrl" : "http://10.40.3.113:8080/render-ws/v1", + "owner" : "trautmane", "project" : "w60_serial_290_to_299", + "stackIdWithZ" : { "projectPattern": "^w6._serial_..._to_...$", "zValuesPerBatch": 1 } + }, + "pipelineStackGroups": { + "raw": { "stackPattern": "^w.._s..._r.._d00$" }, + "aligned": { "stackPattern": "^w.._s..._r.._d00_align$" }, + "intensityCorrected": { "stackPattern": "^w.._s..._r.._d00_align_ic$" } + }, + "pipelineSteps": [ + "DERIVE_TILE_MATCHES", + "PATCH_MFOV_MONTAGE_MATCHES", + "FIND_UNCONNECTED_MFOVS" + ], + "matchRunList": [ + { + "runName": "montageMfovRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludeSameMfovNeighbors": true, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montageMfovPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.2, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.3, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "montageRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montagePass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.8, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 140, "clipWidth": 80 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "crossRun", + "matchCommon": { + "maxPairsPerStackBatch" : 50, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor" : 0.1, "useRowColPositions" : false, + "zNeighborDistance" : 3, "excludeCornerNeighbors" : false, + "excludeCompletelyObscuredTiles" : true, "excludeSameLayerNeighbors" : true, "excludeSameSectionNeighbors" : false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "crossPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 40, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.14, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 20, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.22, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "AGGREGATED_CONSENSUS_SETS", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 10, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 1 + } + ] + } + ], + "mfovMontagePatch": { + "sameLayerDerivedMatchWeight" : 0.15, + "crossLayerDerivedMatchWeight" : 0.1, + "secondPassDerivedMatchWeight" : 0.05, + "xyNeighborFactor" : 0.6, + "numberOfMFOVsPerBatch": 19 + }, + "unconnectedCrossMfov": { + "minPairsForConnection": 6, + "numberOfStacksPerBatch": 20 + } +} diff --git a/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match.json b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match.json new file mode 100644 index 000000000..53fdef7c6 --- /dev/null +++ b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.match.json @@ -0,0 +1,143 @@ +{ + "multiProject": { + "baseDataUrl" : "http://10.40.3.113:8080/render-ws/v1", + "owner" : "trautmane", "project" : "w60_serial_290_to_299", + "stackIdWithZ" : { "projectPattern": "^w6._serial_..._to_...$", "zValuesPerBatch": 1 } + }, + "pipelineStackGroups": { + "raw": { "stackPattern": "^w.._s..._r.._d00$" }, + "aligned": { "stackPattern": "^w.._s..._r.._d00_align$" }, + "intensityCorrected": { "stackPattern": "^w.._s..._r.._d00_align_ic$" } + }, + "pipelineSteps": [ + "DERIVE_TILE_MATCHES" + ], + "matchRunList": [ + { + "runName": "montageMfovRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludeSameMfovNeighbors": true, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montageMfovPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.2, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.3, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "montageRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montagePass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.8, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 140, "clipWidth": 80 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "crossRun", + "matchCommon": { + "maxPairsPerStackBatch" : 50, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor" : 0.1, "useRowColPositions" : false, + "zNeighborDistance" : 3, "excludeCornerNeighbors" : false, + "excludeCompletelyObscuredTiles" : true, "excludeSameLayerNeighbors" : true, "excludeSameSectionNeighbors" : false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "crossPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 40, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.14, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 20, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.22, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "AGGREGATED_CONSENSUS_SETS", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 10, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 1 + } + ] + } + ] +} diff --git a/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.mip-match-patch-check.json b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.mip-match-patch-check.json new file mode 100644 index 000000000..1b277e7e3 --- /dev/null +++ b/render-ws-spark-client/src/main/resources/multisem/wafer_60/pipeline_json/01_match/pipe.01.mip-match-patch-check.json @@ -0,0 +1,164 @@ +{ + "multiProject": { + "baseDataUrl" : "http://10.40.3.113:8080/render-ws/v1", + "owner" : "trautmane", "project" : "w60_serial_290_to_299", + "stackIdWithZ" : { "projectPattern": "^w6._serial_..._to_...$", "zValuesPerBatch": 1 } + }, + "pipelineStackGroups": { + "raw": { "stackPattern": "^w.._s..._r.._d00$" }, + "aligned": { "stackPattern": "^w.._s..._r.._d00_align$" }, + "intensityCorrected": { "stackPattern": "^w.._s..._r.._d00_align_ic$" } + }, + "pipelineSteps": [ + "GENERATE_MIPMAPS", + "DERIVE_TILE_MATCHES", + "PATCH_MFOV_MONTAGE_MATCHES", + "FIND_UNCONNECTED_MFOVS" + ], + "mipmap" : { + "rootDirectory" : "/nrs/hess/data/hess_wafers_60_61/mipmaps", + "minLevel" : 1, + "maxLevel" : 3, + "format" : "tiff", + "forceGeneration" : false + }, + "matchRunList": [ + { + "runName": "montageMfovRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludeSameMfovNeighbors": true, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montageMfovPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.2, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.3, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + }, + { + "stageName": "montageMfovPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 1300, "clipWidth": 1300 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "montageRun", + "matchCommon": { + "maxPairsPerStackBatch" : 30, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, "useRowColPositions": false, + "zNeighborDistance": 0, "excludeCornerNeighbors": false, + "excludeCompletelyObscuredTiles": false, "excludeSameLayerNeighbors": false, "excludeSameSectionNeighbors": false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montagePass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.25, "steps": 3 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 20.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 25, "matchModelType": "TRANSLATION", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.8, "renderWithFilter": true, "renderWithoutMask": false }, + "featureRenderClip": { "clipHeight": 140, "clipWidth": 80 }, + "geometricDescriptorAndMatch": { "gdEnabled": false } + } + ] + }, + { + "runName": "crossRun", + "matchCommon": { + "maxPairsPerStackBatch" : 50, + "featureStorage": { "maxFeatureSourceCacheGb": 6 }, + "maxPeakCacheGb" : 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor" : 0.1, "useRowColPositions" : false, + "zNeighborDistance" : 3, "excludeCornerNeighbors" : false, + "excludeCompletelyObscuredTiles" : true, "excludeSameLayerNeighbors" : true, "excludeSameSectionNeighbors" : false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "crossPass1", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 40, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.14, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass2", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", "matchFullScaleCoverageRadius": 150.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 50.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 20, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.22, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass3", + "featureExtraction": { "fdSize": 4, "maxScale": 1.0, "minScale": 0.125, "steps": 5 }, + "featureMatchDerivation": { + "matchFilter": "AGGREGATED_CONSENSUS_SETS", "matchFullScaleCoverageRadius": 300.0, "matchIterations": 1000, "matchMaxEpsilonFullScale": 5.0, "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, "matchMinInlierRatio": 0.0, "matchMinNumInliers": 10, "matchModelType": "RIGID", "matchRod": 0.92 + }, + "featureRender": { "renderScale": 0.4, "renderWithFilter": true, "renderWithoutMask": false }, + "geometricDescriptorAndMatch": { "gdEnabled": false }, + "maxNeighborDistance": 1 + } + ] + } + ], + "mfovMontagePatch": { + "sameLayerDerivedMatchWeight" : 0.15, + "crossLayerDerivedMatchWeight" : 0.1, + "secondPassDerivedMatchWeight" : 0.05, + "xyNeighborFactor" : 0.6, + "numberOfMFOVsPerBatch": 19 + }, + "unconnectedCrossMfov": { + "minPairsForConnection": 6, + "numberOfStacksPerBatch": 20 + } +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java index 02db13956..04e655fa5 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java @@ -44,7 +44,8 @@ public void testBuildStepClients() private AlignmentPipelineParameters loadTestParameters() throws IOException { final AlignmentPipelineParameters pipelineParameters = - AlignmentPipelineParameters.fromJsonFile("src/test/resources/pipeline/msem_alignment_pipeline.json"); + AlignmentPipelineParameters.fromJsonFile("src/test/resources/pipeline/msem_alignment_pipeline.json", + null); Assert.assertNotNull("deserialized parameters are null", pipelineParameters); return pipelineParameters; }