Merge pull request #320 from INCATools/odk1222

WIP: New ODK 1222

Changes.md

@@ -1,4 +1,29 @@
-# v1.2.21 (2 January 2020)
+# v1.2.22 (23 February 2020)
+- New ROBOT (v. 1.6.0)
+- Bugfixes: 
+  - Small fixes to [update_repo method](https://github.com/INCATools/ontology-development-kit/issues/305)
+  - the sparql query for [extracting labeled subsets](https://github.com/INCATools/ontology-development-kit/issues/146) is now fixed
+  - Some unneeded files (ontologyterms.txt) are removed from the default template.
+  - a few more files are added to the [.gitignore file](https://github.com/INCATools/ontology-development-kit/issues/317)
+  - The [catalog file is restructured](https://github.com/INCATools/ontology-development-kit/issues/105) to preclude Protege from changing it in unwanted ways.
+  - the license in the ontology metadata is now required to be an IRI rather than a string.
+  - The odk.py should now be executable in a [non-docker context](https://github.com/INCATools/ontology-development-kit/issues/312)
+  - JSON versions of imports are not anymore generated. OBO versions are only generated if OBO is used as the edit file serialisation.
+  - PAT=false now skips pattern download
+- Fixes to DOSDP pipeline
+  - DOSDP tools is updated to version 0.14. This version comes with some great changes, including:
+    - a mode in which multiple patterns can be compiled at once (batch mode)
+    - we finally have have optional columns and column values!
+  - external.txt can now be empty
+  - instead of abnormalAnatomicalEntity, an example.yaml is added to the default pattern directory
+- Infrastructure (only relevant to ODK developers):
+  - The test framework for ODK is now more dynamic: 
+    1. Travis now correctly builds the ODK from scratch and runs the tests against the new build version
+    1. Instead of adding new tests to the Makefile, you can simply now drop a new YAML file into the tests/ directory.
+  - ODK Dockerfile has been optimised a bit for size (avoiding superfluous chmod calls)
+  - the yaml loader in the ODK is now replaced by a safe loader
+
+# v1.2.21 (23 January 2020)
 - switched the Docker base image to ubuntu:18.04
 - Added ammonite to the container
 

Dockerfile

@@ -3,16 +3,9 @@
 FROM ubuntu:18.04
 LABEL maintainer="[email protected]" 
 
-ARG ODK_VERSION=0.0.0
-ENV ODK_VERSION ${ODK_VERSION}
+### 2. Get Java, Python and all required system libraries (version control etc)
 ENV JAVA_HOME="/usr/lib/jvm/java-1.8-openjdk"
 #ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
-ENV ROBOT v1.5.0
-ARG ROBOT_JAR=https://github.com/ontodev/robot/releases/download/$ROBOT/robot.jar
-ENV ROBOT_JAR ${ROBOT_JAR}
-ENV DOSDPVERSION=0.14
-
-### 2. Get Java, Python and all required system libraries (version control etc)
 
 RUN apt-get update && apt-get upgrade -y \
  && apt-get install -y software-properties-common \
@@ -43,7 +36,10 @@ RUN pip3 install -r requirements.txt && pip3 install jsonschema ruamel.yaml requ
 # by composing existing Dockerfiles, or by obtaining directly from maven
 RUN wget http://build.berkeleybop.org/userContent/owltools/owltools -O /tools/owltools && \
     wget http://build.berkeleybop.org/userContent/owltools/ontology-release-runner -O /tools/ontology-release-runner && \
-    wget http://build.berkeleybop.org/userContent/owltools/owltools-oort-all.jar -O /tools/owltools-oort-all.jar 
+    wget http://build.berkeleybop.org/userContent/owltools/owltools-oort-all.jar -O /tools/owltools-oort-all.jar && \
+    chmod +x /tools/owltools && \
+    chmod +x /tools/ontology-release-runner && \
+    chmod +x /tools/owltools-oort-all.jar
 
 ###### Konclude, the DL reasoner ######
 RUN wget https://github.com/konclude/Konclude/releases/download/v0.6.2-845/Konclude-v0.6.2-845-LinuxAlpine-x64-GCC8.3.0-Static-Qt-5.13.zip -O /tools/konclude.zip && \
@@ -56,16 +52,22 @@ RUN wget https://github.com/konclude/Konclude/releases/download/v0.6.2-845/Koncl
     chmod +x /tools/Konclude
 
 ###### ROBOT ######
+ENV ROBOT v1.6.0
+ARG ROBOT_JAR=https://github.com/ontodev/robot/releases/download/$ROBOT/robot.jar
+ENV ROBOT_JAR ${ROBOT_JAR}
+RUN pwd
 RUN wget $ROBOT_JAR -O /tools/robot.jar && \
     wget https://raw.githubusercontent.com/ontodev/robot/$ROBOT/bin/robot -O /tools/robot && \
-    chmod +x /tools/*
+    chmod +x /tools/robot && \
+    chmod +x /tools/robot.jar
 
 # Avoid repeated downloads of script dependencies by mounting the local coursier cache: 
 # docker run -v $HOME/.coursier/cache/v1:/tools/.coursier-cache ...
 ENV COURSIER_CACHE "/tools/.coursier-cache"
 
 ###### FASTOBO ######
-RUN wget https://dl.bintray.com/fastobo/fastobo-validator/stable/fastobo_validator-x86_64-linux-musl.tar.gz -O- | tar xzC /tools
+RUN wget https://dl.bintray.com/fastobo/fastobo-validator/stable/fastobo_validator-x86_64-linux-musl.tar.gz -O- | tar xzC /tools && \
+chmod +x /tools/fastobo-validator
 
 ##### Ammonite #####
 RUN (echo "#!/usr/bin/env sh" \
@@ -75,17 +77,23 @@ RUN (echo "#!/usr/bin/env sh" \
 RUN amm /dev/null
 
 ###### DOSDPTOOLS ######
+ENV DOSDPVERSION=0.14
 ENV PATH "/tools/dosdp-tools/bin:$PATH"
 RUN wget -nv https://github.com/INCATools/dosdp-tools/releases/download/v$DOSDPVERSION/dosdp-tools-$DOSDPVERSION.tgz \
 && tar -zxvf dosdp-tools-$DOSDPVERSION.tgz \
 && mv dosdp-tools-$DOSDPVERSION /tools/dosdp-tools \
 && wget --no-check-certificate https://raw.githubusercontent.com/INCATools/dead_simple_owl_design_patterns/master/src/simple_pattern_tester.py -O /tools/simple_pattern_tester.py \
-&& chmod +x /tools/*
+&& chmod +x /tools/dosdp-tools \
+&& chmod +x /tools/simple_pattern_tester.py
 
 ### 5. Install ODK
 
+ARG ODK_VERSION=0.0.0
+ENV ODK_VERSION ${ODK_VERSION}
+
 COPY template/ /tools/templates/
-COPY odk/ /tools/
+COPY odk/ /tools/ 
+RUN chmod +x /tools/*.py
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 

Makefile

@@ -9,6 +9,8 @@ CMD = ./odk/odk.py seed
 
 EMAIL_ARGS=
 
+.PHONY: .FORCE
+
 custom_tests: test_no_yaml_dependencies_none test_no_yaml_dependencies_ro_pato test_no_yaml_dependencies_ro_pato_cl test_go_mini
 
 test_no_yaml_dependencies_none:
@@ -24,16 +26,18 @@ test_go_mini:
 	$(CMD) -c -C examples/go-mini/project.yaml -s examples/go-mini/go-edit.obo -D target/go-mini
 
 TESTS = $(notdir $(wildcard tests/*.yaml))
-tests: custom_tests $(TESTS)
-
-tests/*.yaml:
+TEST_FILES = $(foreach n,$(TESTS), tests/$(n))
+test: custom_tests $(TEST_FILES)
+	echo "All tests passed successfully!"
+
+tests/*.yaml: .FORCE
 	$(CMD) -c -C $@
 
 schema/project-schema.json:
 	./odk/odk.py dump-schema > $@
 
 # Building docker image
-VERSION = "v1.2.21" 
+VERSION = "v1.2.22" 
 IM=obolibrary/odkfull
 DEV=obolibrary/odkdev
 
@@ -71,7 +75,7 @@ docker-publish: docker-build
 docker-test: docker-build-use-cache
 	docker images | grep odkfull &&\
 	make test CMD=./seed-via-docker.sh
-	
+
 docker-test-no-build:
 	docker images | grep odkfull &&\
 	make test CMD=./seed-via-docker.sh
@@ -83,3 +87,5 @@ docker-test-dev-no-build:
 docker-publish-all: docker-publish
 	(cd docker/osklite && make publish VERSION=$(VERSION))
 
+clean-tests:
+	rm -rf target/*

odk/odk.py

@@ -364,8 +364,8 @@ class OntologyProject(JsonSchemaMixin):
     primary_release : str = 'full'
     """Which release file should be published as the primary release artefact, i.e. foo.owl"""
 
-    license : str = 'Unspecified'
-    """Which license is ontology supplied under; ideally CC-BY."""
+    license : str = 'https://creativecommons.org/licenses/unspecified'
+    """Which license is ontology supplied under - must be an IRI."""
 
     description : str = 'None'
     """Provide a short description of the ontology"""
@@ -510,7 +510,7 @@ def load_config(self,
         else:
             with open(config_file, 'r') as stream:
                 try:
-                    obj = yaml.load(stream)
+                    obj = yaml.load(stream, Loader=yaml.FullLoader)
                 except yaml.YAMLError as exc:
                     print(exc)
             project = from_dict(data_class=OntologyProject, data=obj)

template/_dynamic_files.jinja2

@@ -23,7 +23,7 @@
 {% if project.edit_format == "obo" %}
 {% if project.import_group is defined -%}
 {% for imp in project.import_group.products %}
-import: http://purl.obolibrary.org/obo/{{ project.id }}/imports/{{ imp.id }}_import.obo
+import: http://purl.obolibrary.org/obo/{{ project.id }}/imports/{{ imp.id }}_import.owl
 {% endfor %}
 {% endif %}
 {%- if project.use_dosdps %}
@@ -33,6 +33,9 @@ import: http://purl.obolibrary.org/obo/{{ project.id }}/patterns/pattern.owl
 {% endif -%}
 {% endif %}
 ontology: {{project.id}}
+property_value: http://purl.org/dc/elements/1.1/description "{{ project.description }}" xsd:string
+property_value: http://purl.org/dc/elements/1.1/title "{{ project.title }}" xsd:string
+property_value: http://purl.org/dc/terms/license {{ project.license }}
 
 [Term]
 id: {{ project.id | upper }}:00000000
@@ -64,7 +67,7 @@ Import(<http://purl.obolibrary.org/obo/{{ project.id }}/patterns/pattern.owl>)
 {% endif %}
 
 Annotation(dce:title "{{ project.title }}")
-Annotation(dcterms:license "{{ project.license }}")
+Annotation(dcterms:license <{{ project.license }}>)
 Annotation(dce:description "{{ project.description }}")
 
 Declaration(Class(<http://purl.obolibrary.org/obo/{{ project.id | upper }}_00000000>))
@@ -307,12 +310,12 @@ entries:
     Example pattern implementation TSV
 #}
 {%- if project.use_dosdps %}
-^^^ src/patterns/data/default/abnormalAnatomicalEntity.tsv
-defined_class	defined_class_label	anatomical_entity	anatomical_entity_label
+^^^ src/patterns/data/default/README.md
+Documentation of the Default DOSDP Pipeline
 {%- if project.pattern_pipelines_group is defined %}
 {%- for pipeline in project.pattern_pipelines_group.products %}
-^^^ src/patterns/data/{{ pipeline.id }}/abnormalAnatomicalEntity.tsv
-defined_class	defined_class_label	anatomical_entity	anatomical_entity_label
+^^^ src/patterns/data/{{ pipeline.id }}/README.md
+# Documentation of the {{ pipeline.id }} DOSDP Pipeline
 {%- endfor %}
 {%- endif %}
 {%- endif %}