Test OAK Validation using ODK

.gitignore

@@ -17,6 +17,7 @@ src/ontology/cl-simple.*
 src/ontology/cl.json
 src/ontology/cl.obo
 src/ontology/cl.owl
+src/ontology/cl.db
 src/ontology/dosdp-tools.log
 src/ontology/ed_definitions_merged.owl
 src/ontology/imports/*_terms_combined.txt
@@ -46,6 +47,7 @@ src/patterns/pattern_owl_seed.txt
 src/ontology/ontologyterms.txt
 src/ontology/simple_seed.txt
 src/ontology/reports/*
+!src/ontology/reports/obo-profile.tsv
 src/ontology/cl-hipc.owl
 site/
 src/ontology/cl-check.obo

docs/odk-workflows/RepoManagement.md

@@ -2,7 +2,7 @@
 
 ## Updating your ODK repository
 
-Your ODK repositories configuration is managed in `src/ontology/cl-odk.yaml`. Once you have made your changes, you can run the following to apply your changes to the repository:
+Your ODK repositories configuration is managed in `src/ontology/cl-odk.yaml`. The [ODK Project Configuration Schema](https://github.com/INCATools/ontology-development-kit/blob/master/docs/project-schema.md) defines all possible parameters that can be used in this config YAML. Once you have made your changes, you can run the following to apply your changes to the repository:
 
 
 ```

src/ontology/Makefile

@@ -1,14 +1,17 @@
 # ----------------------------------------
 # Makefile for cl
 # Generated using ontology-development-kit
-# ODK Version: v1.4.3
+# ODK Version: v1.5
 # ----------------------------------------
 # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use cl.Makefile instead
 
 
 # ----------------------------------------
 # More information: https://github.com/INCATools/ontology-development-kit/
 
+# Fingerprint of the configuration file when this Makefile was last generated
+CONFIG_HASH=                34b786df5efb6fe2e44e1bc2c6f078ac6aec7b8ba7d6c3ba403506ef0b231484
+
 
 # ----------------------------------------
 # Standard Constants
@@ -43,7 +46,7 @@ REPORT_PROFILE_OPTS =
 OBO_FORMAT_OPTIONS =        
 SPARQL_VALIDATION_CHECKS =  equivalent-classes owldef-self-reference nolabels pmid-not-dbxref obsolete-replaced_by obsolete-alt-id orcid-contributor illegal-annotation-property label-synonym-polysemy 
 SPARQL_EXPORTS =            basic-report 
-ODK_VERSION_MAKEFILE =      v1.4.3
+ODK_VERSION_MAKEFILE =      v1.5
 
 TODAY ?=                    $(shell date +%Y-%m-%d)
 OBODATE ?=                  $(shell date +'%d:%m:%Y %H:%M')
@@ -63,6 +66,11 @@ FORMATS = $(sort  owl obo json owl)
 FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
 RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full $(ONT)-simple $(ONT)-basic $(ONT)-non-classified cl-plus )
 
+ifeq ($(ODK_DEBUG),yes)
+ODK_DEBUG_FILE = $(TMPDIR)/debug.log
+SHELL = $(SCRIPTSDIR)/run-command.sh
+endif
+
 # ----------------------------------------
 # Top-level targets
 # ----------------------------------------
@@ -73,12 +81,16 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full $(ONT)-simple $(ONT)-basic $(
 all: all_odk
 
 .PHONY: all_odk
-all_odk: odkversion test all_assets
+all_odk: odkversion config_check test all_assets
 
 .PHONY: test
 test: odkversion dosdp_validation reason_test sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
 	echo "Finished running all tests successfully."
 
+.PHONY: test
+test_fast:
+	$(MAKE_FAST) test
+
 .PHONY: release_diff
 release_diff: $(REPORTDIR)/release-diff.md
 
@@ -92,10 +104,50 @@ odkversion:
 	echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
         not the version of the ODK you are running)" &&\
 	echo "ROBOT version (ODK): " && $(ROBOT) --version
+.PHONY: config_check
+config_check:
+	@if [ "$$(sha256sum $(ONT)-odk.yaml | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \
+		echo "Repository is up-to-date." ; else \
+		echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'make update_repo'." ; fi
+
 
 $(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
 	mkdir -p $@
 
+# ----------------------------------------
+# ODK-managed ROBOT plugins
+# ----------------------------------------
+
+# Make sure ROBOT knows where to find plugins
+export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins
+
+# Override this rule in cl.Makefile to install custom plugins
+.PHONY: custom_robot_plugins
+custom_robot_plugins:
+
+
+.PHONY: extra_robot_plugins
+extra_robot_plugins: 
+
+
+# Install all ROBOT plugins to the runtime plugins directory
+.PHONY: all_robot_plugins
+all_robot_plugins: $(foreach plugin,$(notdir $(wildcard /tools/robot-plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+		   $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+		   custom_robot_plugins extra_robot_plugins  \
+
+# Default rule to install plugins
+$(ROBOT_PLUGINS_DIRECTORY)/%.jar:
+	@mkdir -p $(ROBOT_PLUGINS_DIRECTORY)
+	@if [ -f ../../plugins/$*.jar ]; then        \
+		ln ../../plugins/$*.jar $@ ;         \
+	elif [ -f /tools/robot-plugins/$*.jar ]; then \
+		cp /tools/robot-plugins/$*.jar $@ ;  \
+	fi
+
+# Specific rules for supplementary plugins defined in configuration
+
+
 # ----------------------------------------
 # Release assets
 # ----------------------------------------
@@ -185,10 +237,10 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt
 
 SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql)
 
-sparql_test:  $(EDIT_PREPROCESSED) catalog-v001.xml | $(REPORTDIR)
+sparql_test:  $(EDIT_PREPROCESSED) | $(REPORTDIR)
 ifneq ($(SPARQL_VALIDATION_QUERIES),)
 
-	$(ROBOT) verify  --catalog catalog-v001.xml -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
+	$(ROBOT) verify -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
 endif
 
 # ----------------------------------------
@@ -201,6 +253,29 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
 $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
 	$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
 
+
+# ----------------------------------------
+# OAK Validation
+# ----------------------------------------
+
+$(ONT)-simple.db: $(ONT)-simple.owl
+	semsql make $@
+	rm $(ONT)-simple-relation-graph.tsv.gz
+
+
+$(TMPDIR)/obo-profile.yaml:
+	cp obo-profile.yaml $@
+
+
+$(REPORTDIR)/obo-profile.tsv: $(ONT)-simple.db $(TMPDIR)/obo-profile.yaml
+	runoak validate-multiple $< --schema $(TMPDIR)/obo-profile.yaml --output $@ --cutoff 5000
+	awk 'NR==1 {print} NR!=1 && $$2~/CL:/' $@ > [email protected] && mv [email protected] $@
+
+
+.PHONY: oak_validation
+oak_validation: 
+	$(MAKE_FAST)  $(REPORTDIR)/obo-profile.tsv
+
 # ----------------------------------------
 # Release assets
 # ----------------------------------------
@@ -218,15 +293,15 @@ RELEASE_ASSETS = \
   $(SUBSET_FILES)
 
 .PHONY: all_assets
-all_assets: $(ASSETS) 
+all_assets: $(ASSETS) check_rdfxml_assets
 
 .PHONY: show_assets
 show_assets:
 	echo $(ASSETS)
 	du -sh $(ASSETS)
 
 check_rdfxml_%: %
-	@check-rdfxml $<
+	@check-rdfxml  $<
 
 .PHONY: check_rdfxml_assets
 check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl)
@@ -399,9 +474,9 @@ recreate-%:
 no-mirror-recreate-%:
 	$(MAKE) COMP=true IMP=false IMP_LARGE=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl -B
 
-$(COMPONENTSDIR)/%: | $(COMPONENTSDIR)
-	touch $@
-.PRECIOUS: $(COMPONENTSDIR)/%
+$(COMPONENTSDIR)/%.owl: | $(COMPONENTSDIR)
+	test -f $@ || touch $@
+.PRECIOUS: $(COMPONENTSDIR)/%.owl
 
 
 
@@ -453,77 +528,85 @@ IMP=true # Global parameter to bypass import generation
 MIR=true # Global parameter to bypass mirror generation
 IMP_LARGE=true # Global parameter to bypass handling of large imports
 
+ifeq ($(strip $(MIR)),true)
 
 
 ## ONTOLOGY: pr
 .PHONY: mirror-pr
 .PRECIOUS: $(MIRRORDIR)/pr.owl
+ifeq ($(strip $(IMP_LARGE)),true)
 mirror-pr: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ] && [ $(IMP_LARGE) = true ]; then $(ROBOT) convert -I https://raw.githubusercontent.com/obophenotype/pro_obo_slim/master/pr_slim.owl -o [email protected] && \
-		$(ROBOT) remove -i [email protected] --base-iri $(URIBASE)/PR --axioms external --preserve-structure false --trim false -o [email protected] &&\
-		mv [email protected] $(TMPDIR)/[email protected]; fi
+	$(ROBOT) remove -I https://raw.githubusercontent.com/obophenotype/pro_obo_slim/master/pr_slim.owl --base-iri $(OBOBASE)/PR  --axioms external --preserve-structure false --trim false -o $(TMPDIR)/[email protected]
+else
+mirror-pr:
+	@echo "Not refreshing pr because refreshing large imports is disabled (IMP_LARGE=$(IMP_LARGE))."
+endif
 
 
 ## ONTOLOGY: go
 .PHONY: mirror-go
 .PRECIOUS: $(MIRRORDIR)/go.owl
 mirror-go: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/go/go-base.owl --create-dirs -o $(MIRRORDIR)/go.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(MIRRORDIR)/go.owl -o $@.tmp.owl && mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/go/go-base.owl --create-dirs -o $(TMPDIR)/go-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/go-download.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: uberon
 .PHONY: mirror-uberon
 .PRECIOUS: $(MIRRORDIR)/uberon.owl
 mirror-uberon: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/uberon/uberon-base.owl --create-dirs -o $(MIRRORDIR)/uberon.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(MIRRORDIR)/uberon.owl -o $@.tmp.owl && mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/uberon/uberon-base.owl --create-dirs -o $(TMPDIR)/uberon-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/uberon-download.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: ro
 .PHONY: mirror-ro
 .PRECIOUS: $(MIRRORDIR)/ro.owl
 mirror-ro: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/ro/ro-base.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl && mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/ro/ro-base.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: pato
 .PHONY: mirror-pato
 .PRECIOUS: $(MIRRORDIR)/pato.owl
 mirror-pato: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/pato/pato-base.owl --create-dirs -o $(MIRRORDIR)/pato.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(MIRRORDIR)/pato.owl -o $@.tmp.owl && mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/pato/pato-base.owl --create-dirs -o $(TMPDIR)/pato-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/pato-download.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: ncbitaxon
 .PHONY: mirror-ncbitaxon
 .PRECIOUS: $(MIRRORDIR)/ncbitaxon.owl
 mirror-ncbitaxon: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl -o [email protected] &&\
-		mv [email protected] $(TMPDIR)/[email protected]; fi
+	$(ROBOT) convert -I http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl -o $(TMPDIR)/[email protected]
 
 
 ## ONTOLOGY: omo
 .PHONY: mirror-omo
 .PRECIOUS: $(MIRRORDIR)/omo.owl
 mirror-omo: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/omo.owl --create-dirs -o $(MIRRORDIR)/omo.owl --retry 4 --max-time 200 &&\
-		$(ROBOT) convert -i $(MIRRORDIR)/omo.owl -o [email protected] &&\
-		mv [email protected] $(TMPDIR)/[email protected]; fi
+	curl -L $(OBOBASE)/omo.owl --create-dirs -o $(TMPDIR)/omo-download.owl --retry 4 --max-time 200 && \
+	$(ROBOT) convert -i $(TMPDIR)/omo-download.owl -o $(TMPDIR)/[email protected]
 
 ALL_MIRRORS = $(patsubst %, $(MIRRORDIR)/%.owl, $(IMPORTS))
 MERGE_MIRRORS = true
 
+ifeq ($(strip $(MERGE_MIRRORS)),true)
 $(MIRRORDIR)/merged.owl: $(ALL_MIRRORS)
-	if [ $(IMP) = true ] && [ $(MERGE_MIRRORS) = true ]; then $(ROBOT) merge $(patsubst %, -i %, $^) -o $@; fi
+	$(ROBOT) merge $(patsubst %, -i %, $^)  remove --axioms equivalent --preserve-structure false -o $@
 .PRECIOUS: $(MIRRORDIR)/merged.owl
+endif
 
 
 $(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
-	if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
+	if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
 		cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
 
+else # MIR=false
+$(MIRRORDIR)/%.owl:
+	@echo "Not refreshing $@ because the mirrorring pipeline is disabled (MIR=$(MIR))."
+endif
 
 
 
@@ -621,7 +704,7 @@ DOSDP_TERM_FILES_DEFAULT = $(patsubst %.tsv, $(PATTERNDIR)/data/default/%.txt, $
 DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(patsubst %.tsv,%, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv))))
 
 $(DOSDP_OWL_FILES_DEFAULT): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(ALL_PATTERN_FILES)
-	if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ]; then $(DOSDPT) generate --catalog=catalog-v001.xml \
+	if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \
     --infile=$(PATTERNDIR)/data/default/ --template=$(PATTERNDIR)/dosdp-patterns --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \
     --ontology=$< --obo-prefixes=true --outfile=$(PATTERNDIR)/data/default; fi
 
@@ -635,7 +718,7 @@ DOSDP_TSV_FILES_CLUSTERING = $(wildcard $(PATTERNDIR)/data/clustering/*.tsv)
 DOSDP_PATTERN_NAMES_CLUSTERING = $(strip $(patsubst %.tsv,%, $(notdir $(wildcard $(PATTERNDIR)/data/clustering/*.tsv))))
 
 $(DOSDP_OWL_FILES_CLUSTERING): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_CLUSTERING) $(ALL_PATTERN_FILES)
-	if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_CLUSTERING}" ]; then $(DOSDPT) generate --catalog=catalog-v001.xml \
+	if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_CLUSTERING}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \
     --infile=$(PATTERNDIR)/data/clustering --template=$(PATTERNDIR)/dosdp-patterns/ --batch-patterns="$(DOSDP_PATTERN_NAMES_CLUSTERING)" \
     --ontology=$< --obo-prefixes=true --outfile=$(PATTERNDIR)/data/clustering; fi
 # Generate template file seeds
@@ -739,9 +822,13 @@ ROBOT_RELEASE_IMPORT_MODE=$(ROBOT) remove --input $< --select imports --trim fal
 # ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base"
 ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false merge $(patsubst %, -i %, $(OTHER_SRC)) 
 
-# base: All the axioms as they are editted by the editors, excluding reasoning
-$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC)
-	$(ROBOT_RELEASE_IMPORT_MODE_BASE) \
+# base: A version of the ontology that does not include any externally imported axioms.
+$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES)
+	$(ROBOT_RELEASE_IMPORT_MODE) \
+	reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural \
+	relax \
+	reduce -r ELK \
+	remove --base-iri http://purl.obolibrary.org/obo/CP_ --base-iri http://purl.obolibrary.org/obo/CL_ --base-iri http://purl.obolibrary.org/obo/cl# --axioms external --preserve-structure false --trim false \
 	$(SHARED_ROBOT_COMMANDS) \
 	annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
 		--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
@@ -869,6 +956,7 @@ Core commands:
 * prepare_release_fast:	Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns.
 * update_repo:		Update the ODK repository setup using the config file cl-odk.yaml
 * test:			Running all validation tests
+* test_fast:		Runs the test suite, but without updating imports or components
 * odkversion:		Show the current version of the ODK Makefile and ROBOT.
 * clean:		Delete all temporary files
 * help:			Print ODK Usage information
@@ -895,7 +983,7 @@ DOSDP templates
 
 Editor utilities:
 * validate_idranges:	Make sure your ID ranges file is formatted correctly
-* normalize_src:	Load and safe your cl-edit file after you to make sure its serialised correctly
+* normalize_src:	Load and save your cl-edit file after you to make sure its serialised correctly
 * explain_unsat:	If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes
 * validate-all-tsv:	Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files
 * validate-tsv:		Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv.

src/ontology/cl-odk.yaml

@@ -118,4 +118,7 @@ components:
       use_template: True
       templates:
         - cellxgene_subset.tsv
-
+oak_validation:
+  profiles:
+    - id: obo-profile
+      filename: obo-profile.yaml

src/ontology/cl.Makefile

@@ -307,3 +307,4 @@ $(REPORTDIR)/general_cell_types_upper_slim_report.csv: $(TEMPLATEDIR)/general_ce
 $(REPORTDIR)/kidney_upper_slim_report.csv: $(TEMPLATEDIR)/kidney_upper_slim.csv
 	$(eval TERM_ID := $(TERM_kidney))
 	$(COVERAGECMD)
+