monarch-initiative · matentzn · Feb 9, 2024 · Feb 6, 2024 · matentzn · Feb 7, 2024
diff --git a/docs/odk-workflows/ManageDocumentation.md b/docs/odk-workflows/ManageDocumentation.md
@@ -44,3 +44,4 @@ The documentation is _not_ automatically updated from the Markdown, and needs to
    Just make sure you give GitHub 2-5 minutes to build the pages!
 
 
+
diff --git a/src/ontology/Makefile b/src/ontology/Makefile
@@ -1,7 +1,7 @@
 # ----------------------------------------
 # Makefile for mondo-ingest
 # Generated using ontology-development-kit
-# ODK Version: v1.5
+# ODK Version: v1.5-dev
 # ----------------------------------------
 # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use mondo-ingest.Makefile instead
 
@@ -10,7 +10,7 @@
 # More information: https://github.com/INCATools/ontology-development-kit/
 
 # Fingerprint of the configuration file when this Makefile was last generated
-CONFIG_HASH=                cb9c226c830886e98787ddf41795ed278d091ca821b301895ae80ac5429e9379
+CONFIG_HASH=                75f9fac53818e7a8265abd51f2671d660b4e535d51c9226f29b815e013af0edf
 
 
 # ----------------------------------------
@@ -47,7 +47,7 @@ REPORT_PROFILE_OPTS =       --profile $(ROBOT_PROFILE)
 OBO_FORMAT_OPTIONS =        
 SPARQL_VALIDATION_CHECKS =  properties-as-annotation-and-object ordo-mapping-annotations 
 SPARQL_EXPORTS =            basic-report 
-ODK_VERSION_MAKEFILE =      v1.5
+ODK_VERSION_MAKEFILE =      v1.5-dev
 
 TODAY ?=                    $(shell date +%Y-%m-%d)
 OBODATE ?=                  $(shell date +'%d:%m:%Y %H:%M')
@@ -62,6 +62,11 @@ FORMATS = $(sort  owl obo json owl)
 FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
 RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full )
 
+ifeq ($(ODK_DEBUG),yes)
+ODK_DEBUG_FILE = $(TMPDIR)/debug.log
+SHELL = $(SCRIPTSDIR)/run-command.sh
+endif
+
 # ----------------------------------------
 # Top-level targets
 # ----------------------------------------
@@ -105,6 +110,40 @@ config_check:
 $(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
 	mkdir -p $@
 
+# ----------------------------------------
+# ODK-managed ROBOT plugins
+# ----------------------------------------
+
+# Make sure ROBOT knows where to find plugins
+export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins
+
+# Override this rule in mondo-ingest.Makefile to install custom plugins
+.PHONY: custom_robot_plugins
+custom_robot_plugins:
+
+
+.PHONY: extra_robot_plugins
+extra_robot_plugins: 
+
+
+# Install all ROBOT plugins to the runtime plugins directory
+.PHONY: all_robot_plugins
+all_robot_plugins: $(foreach plugin,$(notdir $(wildcard /tools/robot-plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+		   $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+		   custom_robot_plugins extra_robot_plugins  \
+
+# Default rule to install plugins
+$(ROBOT_PLUGINS_DIRECTORY)/%.jar:
+	@mkdir -p $(ROBOT_PLUGINS_DIRECTORY)
+	@if [ -f ../../plugins/$*.jar ]; then        \
+		ln ../../plugins/$*.jar $@ ;         \
+	elif [ -f /tools/robot-plugins/$*.jar ]; then \
+		cp /tools/robot-plugins/$*.jar $@ ;  \
+	fi
+
+# Specific rules for supplementary plugins defined in configuration
+
+
 # ----------------------------------------
 # Release assets
 # ----------------------------------------
@@ -226,15 +265,15 @@ RELEASE_ASSETS = \
   $(SUBSET_FILES)
 
 .PHONY: all_assets
-all_assets: $(ASSETS) 
+all_assets: $(ASSETS) check_rdfxml_assets
 
 .PHONY: show_assets
 show_assets:
 	echo $(ASSETS)
 	du -sh $(ASSETS)
 
 check_rdfxml_%: %
-	@check-rdfxml $<
+	@check-rdfxml  $<
 
 .PHONY: check_rdfxml_assets
 check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl)
@@ -388,8 +427,11 @@ component-download-doid.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://purl.obolibrary.org/obo/doid.owl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/doid.owl:  component-download-doid.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-doid.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-doid.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/doid.owl: component-download-doid.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-doid.owl.owl $(TMPDIR)/component-download-doid.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-doid.owl.owl $(TMPDIR)/component-download-doid.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-doid.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/doid.owl
 
@@ -399,8 +441,11 @@ component-download-gard.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://github.com/monarch-initiative/gard/releases/latest/download/gard.owl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/gard.owl:  component-download-gard.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-gard.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-gard.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/gard.owl: component-download-gard.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-gard.owl.owl $(TMPDIR)/component-download-gard.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-gard.owl.owl $(TMPDIR)/component-download-gard.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-gard.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/gard.owl
 
@@ -410,8 +455,11 @@ component-download-icd10cm.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://data.bioontology.org/ontologies/ICD10CM/submissions/23/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/icd10cm.owl:  component-download-icd10cm.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-icd10cm.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-icd10cm.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/icd10cm.owl: component-download-icd10cm.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-icd10cm.owl.owl $(TMPDIR)/component-download-icd10cm.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-icd10cm.owl.owl $(TMPDIR)/component-download-icd10cm.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-icd10cm.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/icd10cm.owl
 
@@ -421,8 +469,11 @@ component-download-icd10who.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://github.com/monarch-initiative/icd10who/releases/latest/download/icd10who.ttl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/icd10who.owl:  component-download-icd10who.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-icd10who.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-icd10who.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/icd10who.owl: component-download-icd10who.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-icd10who.owl.owl $(TMPDIR)/component-download-icd10who.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-icd10who.owl.owl $(TMPDIR)/component-download-icd10who.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-icd10who.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/icd10who.owl
 
@@ -432,8 +483,11 @@ component-download-ncit.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://purl.obolibrary.org/obo/ncit.owl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/ncit.owl:  component-download-ncit.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-ncit.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-ncit.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/ncit.owl: component-download-ncit.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-ncit.owl.owl $(TMPDIR)/component-download-ncit.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-ncit.owl.owl $(TMPDIR)/component-download-ncit.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-ncit.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/ncit.owl
 
@@ -443,8 +497,11 @@ component-download-omim.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://github.com/monarch-initiative/omim/releases/latest/download/omim.ttl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/omim.owl:  component-download-omim.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-omim.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-omim.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/omim.owl: component-download-omim.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-omim.owl.owl $(TMPDIR)/component-download-omim.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-omim.owl.owl $(TMPDIR)/component-download-omim.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-omim.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/omim.owl
 
@@ -454,8 +511,11 @@ component-download-ordo.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://www.orphadata.org/data/ORDO/ordo_orphanet.owl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/[email protected]; fi
 
-$(COMPONENTSDIR)/ordo.owl:  component-download-ordo.owl
-	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-ordo.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-ordo.owl.owl $@; fi; fi
+$(COMPONENTSDIR)/ordo.owl: component-download-ordo.owl
+	if [ $(COMP) = true ]; then if cmp -s $(TMPDIR)/component-download-ordo.owl.owl $(TMPDIR)/component-download-ordo.owl.tmp.owl ; then echo "Component identical."; \
+      else echo "Component is different, updating." &&\
+		cp $(TMPDIR)/component-download-ordo.owl.owl $(TMPDIR)/component-download-ordo.owl.tmp.owl &&\
+		$(ROBOT) annotate -i $(TMPDIR)/component-download-ordo.owl.owl --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/ordo.owl
 # ----------------------------------------
@@ -466,30 +526,33 @@ IMP=true # Global parameter to bypass import generation
 MIR=true # Global parameter to bypass mirror generation
 IMP_LARGE=true # Global parameter to bypass handling of large imports
 
+ifeq ($(strip $(MIR)),true)
 
 
 ## ONTOLOGY: ro
 .PHONY: mirror-ro
 .PRECIOUS: $(MIRRORDIR)/ro.owl
 mirror-ro: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/ro.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o [email protected] &&\
-		mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/ro.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: omo
 .PHONY: mirror-omo
 .PRECIOUS: $(MIRRORDIR)/omo.owl
 mirror-omo: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/omo.owl --create-dirs -o $(TMPDIR)/omo-download.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(TMPDIR)/omo-download.owl -o [email protected] &&\
-		mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/omo.owl --create-dirs -o $(TMPDIR)/omo-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/omo-download.owl -o $(TMPDIR)/[email protected]
 
 
 $(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
-	if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
+	if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
 		cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
 
+else # MIR=false
+$(MIRRORDIR)/%.owl:
+	@echo "Not refreshing $@ because the mirrorring pipeline is disabled (MIR=$(MIR))."
+endif
 
 
 

diff --git a/src/ontology/run.sh b/src/ontology/run.sh
@@ -14,6 +14,8 @@
 #
 # See README-editors.md for more details.
 
+set -e
+
 if [ -f run.sh.conf ]; then
     . ./run.sh.conf
 fi
@@ -75,6 +77,7 @@ if [ x$ODK_DEBUG = xyes ]; then
     echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps."
     TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb"
 fi
+rm -f tmp/debug.log
 
 VOLUME_BIND=$PWD/../../:/work$ODK_SSH_BIND
 WORK_DIR=/work/src/ontology
@@ -86,14 +89,14 @@ fi
 if [ -n "$USE_SINGULARITY" ]; then
 
     singularity exec --cleanenv $ODK_SINGULARITY_OPTIONS \
-        --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS,SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock,ODK_USER_ID=$ODK_USER_ID,ODK_GROUP_ID=$ODK_GROUP_ID" \
+        --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS,SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock,ODK_USER_ID=$ODK_USER_ID,ODK_GROUP_ID=$ODK_GROUP_ID,ODK_DEBUG=$ODK_DEBUG" \
         --bind $VOLUME_BIND \
         -W $WORK_DIR \
         docker://obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
 else
     BIND_OPTIONS="-v $(echo $VOLUME_BIND | sed 's/,/ -v /')"
     docker run $ODK_DOCKER_OPTIONS $BIND_OPTIONS -w $WORK_DIR \
-        -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -e ODK_USER_ID=$ODK_USER_ID -e ODK_GROUP_ID=$ODK_GROUP_ID \
+        -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -e ODK_USER_ID=$ODK_USER_ID -e ODK_GROUP_ID=$ODK_GROUP_ID -e ODK_DEBUG=$ODK_DEBUG \
         --rm -ti obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
 fi
 

diff --git a/src/scripts/run-command.sh b/src/scripts/run-command.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+ODK_DEBUG_FILE=${ODK_DEBUG_FILE:-tmp/debug.log}
+echo "Command: sh $@" >> $ODK_DEBUG_FILE
+/usr/bin/time -a -o $ODK_DEBUG_FILE -f "Elapsed time: %E\nPeak memory: %M kb" /bin/sh "$@"
diff --git a/src/scripts/update_repo.sh b/src/scripts/update_repo.sh
@@ -29,6 +29,7 @@ mkdir -p $ROOTDIR/.github/workflows
 cp -n target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml
 
 
+
 cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml
Original file line number	Diff line number	Diff line change
Expand Up		@@ -44,3 +44,4 @@ The documentation is _not_ automatically updated from the Markdown, and needs to
		Just make sure you give GitHub 2-5 minutes to build the pages!



matentzn marked this conversation as resolved. Show resolved Hide resolved
Original file line number	Diff line number	Diff line change
Expand Up		@@ -29,6 +29,7 @@ mkdir -p $ROOTDIR/.github/workflows
		cp -n target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml



		cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml


Expand Down