Skip to content

Commit

Permalink
Merge pull request #288 from OBOFoundry/revised
Browse files Browse the repository at this point in the history
COB Revised -- Wait for James to Merge
jamesaoverton authored Jan 23, 2025
2 parents 0787b2f + 4bb8a41 commit 57f13da
Showing 17 changed files with 2,375 additions and 9,662 deletions.
2,913 changes: 175 additions & 2,738 deletions cob-base.owl

Large diffs are not rendered by default.

1,456 changes: 0 additions & 1,456 deletions cob-full.owl

This file was deleted.

1,369 changes: 1,369 additions & 0 deletions cob-root.owl

Large diffs are not rendered by default.

1,065 changes: 460 additions & 605 deletions cob.owl

Large diffs are not rendered by default.

130 changes: 62 additions & 68 deletions cob.tsv

Large diffs are not rendered by default.

7 changes: 0 additions & 7 deletions docs/odk-workflows/RepositoryFileStructure.md
Original file line number Diff line number Diff line change
@@ -16,8 +16,6 @@ These are the current imports in COB

| Import | URL | Type |
| ------ | --- | ---- |
| ro | http://purl.obolibrary.org/obo/ro.owl | custom |
| omo | http://purl.obolibrary.org/obo/omo.owl | mirror |

## Components
Components, in contrast to imports, are considered full members of the ontology. This means that any axiom in a component is also included in the ontology base - which means it is considered _native_ to the ontology. While this sounds complicated, consider this: conceptually, no component should be part of more than one ontology. If that seems to be the case, we are most likely talking about an import. Components are often not needed for ontologies, but there are some use cases:
@@ -26,9 +24,4 @@ Components, in contrast to imports, are considered full members of the ontology.
2. A part of the ontology is managed in ROBOT templates
3. The expressivity of the component is higher than the format of the edit file. For example, people still choose to manage their ontology in OBO format (they should not) missing out on a lot of owl features. They may choose to manage logic that is beyond OBO in a specific OWL component.

These are the components in COB

| Filename | URL |
| -------- | --- |
| cob-annotations.owl | None |
| cob-to-external.owl | None |
76 changes: 4 additions & 72 deletions src/ontology/Makefile
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@
# 6. [Run everything (COB products and release pipeline)](all)

# Fingerprint of the configuration file when this Makefile was last generated
CONFIG_HASH= 5daef41d99e802417bd9ad0f378f72168e2d30f0488f5b2423330cbc561ec834
CONFIG_HASH= 0cb0824fc712dd1bbe3e3067c8a9d6d4212a769da65f9ec96e0020e31e2cafb9


# ----------------------------------------
@@ -62,13 +62,13 @@ OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
VERSION= $(TODAY)
ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --annotation owl:versionInfo $(VERSION)
ANNOTATE_CONVERT_FILE = annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) convert -f ofn --output $@.tmp.owl && mv $@.tmp.owl $@
OTHER_SRC = $(COMPONENTSDIR)/cob-annotations.owl $(COMPONENTSDIR)/cob-to-external.owl
OTHER_SRC =
ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt
EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl

FORMATS = $(sort owl owl)
FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full cob-base-reasoned cob-examples-reasoned cob-native )
RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-root )

ifeq ($(ODK_DEBUG),yes)
ODK_DEBUG_FILE = debug.log
@@ -168,7 +168,7 @@ all_main: $(MAIN_FILES)
# ----------------------------------------


IMPORTS = ro omo
IMPORTS =

IMPORT_ROOTS = $(patsubst %, $(IMPORTDIR)/%_import, $(IMPORTS))
IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl)
@@ -375,16 +375,6 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms_combined.txt

.PRECIOUS: $(IMPORTDIR)/%_import.owl

## Module for ontology: ro

$(IMPORTDIR)/ro_import.owl: $(MIRRORDIR)/ro.owl
echo "ERROR: You have configured your default module type to be custom; this behavior needs to be overwritten in cob.Makefile!" && false
## Module for ontology: omo

$(IMPORTDIR)/omo_import.owl: $(MIRRORDIR)/omo.owl $(IMPORTDIR)/omo_terms_combined.txt
if [ $(IMP) = true ]; then $(ROBOT) merge -i $< query --update ../sparql/preprocess-module.ru --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \
$(ANNOTATE_CONVERT_FILE); fi


.PHONY: refresh-imports
refresh-imports:
@@ -406,39 +396,6 @@ refresh-%:
no-mirror-refresh-%:
$(MAKE) IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B


# ----------------------------------------
# Components
# ----------------------------------------
# Some ontologies contain external and internal components. A component is included in the ontology in its entirety.

COMP=true # Global parameter to bypass component generation

.PHONY: all_components
all_components: $(OTHER_SRC)

.PHONY: recreate-components
recreate-components:
$(MAKE) COMP=true IMP=false MIR=true PAT=true IMP_LARGE=false all_components -B

.PHONY: no-mirror-recreate-components
no-mirror-recreate-components:
$(MAKE) COMP=true IMP=false MIR=false PAT=true IMP_LARGE=false all_components -B

.PHONY: recreate-%
recreate-%:
$(MAKE) COMP=true IMP=false IMP_LARGE=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl -B

.PHONY: no-mirror-recreate-%
no-mirror-recreate-%:
$(MAKE) COMP=true IMP=false IMP_LARGE=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl -B

$(COMPONENTSDIR)/%.owl: | $(COMPONENTSDIR)
test -f $@ || touch $@
.PRECIOUS: $(COMPONENTSDIR)/%.owl



# ----------------------------------------
# Mirroring upstream ontologies
# ----------------------------------------
@@ -450,22 +407,6 @@ IMP_LARGE=true # Global parameter to bypass handling of large imports
ifeq ($(strip $(MIR)),true)


## ONTOLOGY: ro
.PHONY: mirror-ro
.PRECIOUS: $(MIRRORDIR)/ro.owl
mirror-ro: | $(TMPDIR)
curl -L $(OBOBASE)/ro.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o $(TMPDIR)/$@.owl


## ONTOLOGY: omo
.PHONY: mirror-omo
.PRECIOUS: $(MIRRORDIR)/omo.owl
mirror-omo: | $(TMPDIR)
curl -L $(OBOBASE)/omo.owl --create-dirs -o $(TMPDIR)/omo-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/omo-download.owl -o $(TMPDIR)/$@.owl


$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
@@ -549,15 +490,6 @@ $(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES)
relax \
reduce -r ELK \
$(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@

cob-base-reasoned.owl:
echo "ERROR: You have configured a custom release artefact ($@); this release artefact needs to be define in cob.Makefile!" && false

cob-examples-reasoned.owl:
echo "ERROR: You have configured a custom release artefact ($@); this release artefact needs to be define in cob.Makefile!" && false

cob-native.owl:
echo "ERROR: You have configured a custom release artefact ($@); this release artefact needs to be define in cob.Makefile!" && false
# ----------------------------------------
# Debugging Tools
# ----------------------------------------
1,368 changes: 0 additions & 1,368 deletions src/ontology/cob-edit.owl

This file was deleted.

150 changes: 150 additions & 0 deletions src/ontology/cob-edit.tsv

Large diffs are not rendered by default.

17 changes: 2 additions & 15 deletions src/ontology/cob-odk.yaml
Original file line number Diff line number Diff line change
@@ -10,27 +10,14 @@ workflows:
- docs
release_artefacts:
- base
- full
- custom-cob-base-reasoned
- custom-cob-examples-reasoned
- custom-cob-native
- root
import_group:
module_type: filter
annotation_properties:
- rdfs:label
- IAO:0000115
- IAO:0000116
- IAO:0000111
products:
- id: ro
module_type: custom
slme_individuals: exclude
- id: omo
module_type: mirror
components:
products:
- filename: cob-annotations.owl
- filename: cob-to-external.owl
robot_java_args: '-Xmx8G'
custom_makefile_header: |
### Workflow
@@ -47,4 +34,4 @@ custom_makefile_header: |
# 6. [Run everything (COB products and release pipeline)](all)
create_obo_metadata: FALSE
documentation:
documentation_system: mkdocs
documentation_system: mkdocs
119 changes: 44 additions & 75 deletions src/ontology/cob.Makefile
Original file line number Diff line number Diff line change
@@ -10,16 +10,17 @@ SHELL := bash
.SUFFIXES:
.SECONDARY:

COB_TO_EXTERNAL = $(COMPONENTSDIR)/cob-to-external.owl
COB_ANNOTATIONS = $(COMPONENTSDIR)/cob-annotations.owl
COB_EXAMPLES = $(COMPONENTSDIR)/cob-examples.owl
ANNOTATE_ONTOLOGY_METADATA := \
--prefix "dcterms: http://purl.org/dc/terms/" \
--language-annotation dcterms:title "Core Ontology for Biology and Biomedicine" en \
--language-annotation dcterms:description "COB brings together key terms from a wide range of OBO projects to improve interoperability." en \
--link-annotation dcterms:license https://creativecommons.org/publicdomain/zero/1.0/

.PHONY: prepare_release
prepare_release: $(ASSETS) $(PATTERN_RELEASE_FILES) cob.tsv
rsync -R $(RELEASE_ASSETS) cob.tsv $(RELEASEDIR) &&\
rm -f $(CLEANFILES) &&\
rm -f cob.tsv &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"
rm -f $(CLEANFILES) &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"

.PHONY: prepare_cob_products
prepare_cob_products: test
@@ -36,88 +37,56 @@ products/:
$(TMPDIR)/robot.jar: | $(TMPDIR)
if [ $(ROBOT_DOWNLOAD) = true ]; then curl -L -o $@ https://build.obolibrary.io/job/ontodev/job/robot/job/master/lastSuccessfulBuild/artifact/bin/robot.jar; fi


########################################
# -- MAIN RELEASE PRODUCTS --
# -- TEMPLATES --
########################################

# build main release product
REWIRE_PRECEDENCE = PR CHEBI
cob.ttl: components/cob-to-external.tsv cob-native.owl
sssom rewire -I xml -m $< $(patsubst %,--precedence %,$(REWIRE_PRECEDENCE)) cob-native.owl -o $@

cob.owl: cob.ttl
robot merge --include-annotations true -i $< -i ontology-metadata.owl \
annotate --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output $@.tmp.owl && mv $@.tmp.owl $@
.PRECIOUS: cob.owl

cob-native.owl: $(SRC)
$(ROBOT) remove --input $< --select imports --trim false \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output $@.tmp.owl && mv $@.tmp.owl $@

# base file is main cob plus linking axioms
#cob-base.owl: cob.owl $(COB_TO_EXTERNAL)
# $(ROBOT) merge $(patsubst %, -i %, $^) -o $@

$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC)
$(ROBOT_RELEASE_IMPORT_MODE_BASE) \
$(SHARED_ROBOT_COMMANDS) \
annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output $@.tmp.owl && mv $@.tmp.owl $@

cob-base-reasoned.owl: cob-base.owl
$(ROBOT) remove --input $< --select imports --trim false \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output $@.tmp.owl && mv $@.tmp.owl $@

cob-examples-reasoned.owl: cob-base.owl $(COB_EXAMPLES)
$(ROBOT) remove --input $< --select imports --trim false \
merge $(patsubst %, -i %, $^) \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output $@.tmp.owl && mv $@.tmp.owl $@
$(TMPDIR)/cob-%.tsv: $(SCRIPTSDIR)/split-cob-edit.py cob-edit.tsv
$^ $(TMPDIR)

# TSV export (may depend on dev version of robot export)
cob.tsv: cob.owl
$(ROBOT) export -i $< -c "ID|ID [LABEL]|definition|subClassOf [ID NAMED]|subClassOf [LABEL NAMED]|subClassOf [ID ANON]|subClassOf [LABEL ANON]" -e $@
# $(ROBOT) export -i $< --entity-select NAMED -c "ID|ID [LABEL]|definition|subClassOf [ID]|subClassOf [LABEL]|subClassOf [ID ANON]|subClassOf [LABEL ANON]" -e $@
$(ROBOT) export -i $< -c "ID|ID [LABEL]|definition|subClassOf [ID NAMED]|subClassOf [LABEL NAMED]|subClassOf [ID ANON]|subClassOf [LABEL ANON]" -e $@

# -- BRIDGING AXIOMS TO OBO ROOTS --
#
# the source file is cob-to-external.tsv
#
# OWL is generated from this
#

$(TMPDIR)/cob-to-external.sssom.owl: $(COMPONENTSDIR)/cob-to-external.tsv | $(TMPDIR)
sssom convert $< --output-format owl -o $@
########################################
# -- MAIN RELEASE PRODUCTS --
########################################

$(COB_TO_EXTERNAL): $(TMPDIR)/cob-to-external.sssom.owl
$(ROBOT) merge -i $< \
$(ONT)-edit.owl: $(TMPDIR)/cob-root.tsv $(COMPONENTSDIR)/obsolete.tsv
$(ROBOT) template \
$(foreach X,$(filter %.tsv,$^),--template $(X)) \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert -f owl -o $@
$(ANNOTATE_ONTOLOGY_METADATA) \
--output $@.tmp.owl && mv $@.tmp.owl $@

$(TMPDIR)/cob-annotations.ttl: $(COB_TO_EXTERNAL) $(SPARQLDIR)/external-links.rq | $(TMPDIR)
$(ROBOT) query --input $< --query $(word 2,$^) $@
# COB "Full"
$(ONT).owl: $(TMPDIR)/cob-full.tsv $(COMPONENTSDIR)/obsolete.tsv
$(ROBOT) template \
$(foreach X,$(filter %.tsv,$^),--template $(X)) \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE).owl $(ANNOTATE_ONTOLOGY_VERSION) \
$(ANNOTATE_ONTOLOGY_METADATA) \
--output $@.tmp.owl && mv $@.tmp.owl $@

$(COB_ANNOTATIONS): $(TMPDIR)/cob-annotations.ttl
$(ROBOT) annotate --input $< \
--ontology-iri "http://purl.obolibrary.org/obo/cob/$@" \
--annotation owl:versionInfo $(TODAY) \
--output $@
$(ONT)-base.owl: $(ONT).owl $(TMPDIR)/cob-base.tsv $(COMPONENTSDIR)/obsolete.tsv
$(ROBOT) template --input $< \
$(foreach X,$(filter %.tsv,$^),--template $(X)) \
annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
$(ANNOTATE_ONTOLOGY_METADATA) \
--output $@.tmp.owl && mv $@.tmp.owl $@

# This is the custom import: removing all COB related axioms from RO, but otherwise pulling in logical dependencies.
$(ONT)-root.owl: $(TMPDIR)/cob-root.tsv $(COMPONENTSDIR)/obsolete.tsv
$(ROBOT) template \
$(foreach X,$(filter %.tsv,$^),--template $(X)) \
reason -r HERMIT \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
$(ANNOTATE_ONTOLOGY_METADATA) \
--output $@.tmp.owl && mv $@.tmp.owl $@

$(IMPORTDIR)/ro_import.owl: $(MIRRORDIR)/ro.owl $(IMPORTDIR)/ro_terms_combined.txt
if [ $(IMP) = true ]; then $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \
extract -T $(IMPORTDIR)/ro_terms_combined.txt --copy-ontology-annotations true --force true --individuals exclude --method BOT \
remove --base-iri http://purl.obolibrary.org/obo/COB_ --axioms internal --preserve-structure false --trim false \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \
$(ANNOTATE_CONVERT_FILE); fi

########################################
# -- TESTING --
@@ -151,7 +120,7 @@ cob_test: main_test itest
# main test: should be run via CI on every PR
# this tests COB's internal consistency
.PHONY: main_test
main_test: $(REPORTDIR)/$(SRC)-obo-report.tsv cob.owl cob-base-reasoned.owl cob-examples-reasoned.owl
main_test: $(REPORTDIR)/$(SRC)-obo-report.tsv $(RELEASEDIR)/cob-root.owl
test: main_test

# integration tests: for now, run these on commmand line
226 changes: 0 additions & 226 deletions src/ontology/components/cob-annotations.owl

This file was deleted.

552 changes: 0 additions & 552 deletions src/ontology/components/cob-examples.owl

This file was deleted.

2,338 changes: 0 additions & 2,338 deletions src/ontology/components/cob-to-external.owl

This file was deleted.

142 changes: 0 additions & 142 deletions src/ontology/components/cob-to-external.tsv

This file was deleted.

11 changes: 11 additions & 0 deletions src/ontology/components/obsolete.tsv
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Ontology ID Label Type Definition Deprecated Term Replaced By
ID LABEL TYPE A IAO:0000115 AT owl:deprecated^^xsd:boolean AI IAO:0100001
COB:0000005 obsolete elementary charge owl:Class true
COB:0000014 obsolete macromolecular entity owl:Class true
COB:0000020 obsolete subcellular structure owl:Class true GO:0110165
COB:0000032 obsolete geographical location owl:Class true
COB:0000056 obsolete immaterial anatomical entity owl:Class true UBERON:0000466
COB:0000073 obsolete gene product owl:Class true
COB:0000077 obsolete action specification owl:Class true IAO:0000007
COB:0000116 obsolete cellular membrane owl:Class true
COB:0000122 obsolete physical information carrier owl:Class true
98 changes: 98 additions & 0 deletions src/scripts/split-cob-edit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
#
# Split the cob-edit.tsv file into three ROBOT templates:
# cob-base.tsv cob-full.tsv cob-root.tsv

import argparse
import csv
import os
import re


def initialize_tsv(output_dir, filename, fieldnames, robot_row):
'''Initialize and return a DictWriter for a ROBOT template.'''
path = os.path.join(output_dir, filename)
path_fh = open(path, 'w')
writer = csv.DictWriter(path_fh, fieldnames, delimiter='\t', lineterminator='\n', extrasaction='ignore')
writer.writeheader()
writer.writerow(robot_row)
return writer


def split(input_path, output_dir):
'''Split the COB term template into base, full, and root templates.'''
header_row = None
robot_row, terms, replacements = {}, {}, {}

with open(input_path) as f:
rows = csv.DictReader(f, delimiter='\t')
for row in rows:
if not header_row:
header_row = list(row.keys())
id = row['Ontology ID'].strip()
if id == '':
continue
if id == 'ID':
robot_row = row
continue

# Handle the Replacement columns
# by collecting a dictionary of replacements.
replacement_id = row['Replacement ID'].strip()
replacement_label = row['Replacement Label'].strip()
if replacement_label and replacement_label != '':
# removed exception for non-base IDs... TBD if needed/wanted
replacements[row['Label'].strip()] = {
'ID': id,
'Replacement Label': replacement_label,
'Replacement ID': replacement_id,
}
# Otherwise just add this row to 'terms'.
else:
terms[id] = row

ignore = [
'COB Module', 'COB Module Reason',
'Replacement ID', 'Replacement Label',
'Comment'
]
fieldnames = [x for x in header_row if x not in ignore]

# Apply replacements to the axioms in all the logical columns
axiom_cols = [
'Parent Class', 'Subclass Axiom', 'Equivalent Class Axiom',
'Disjoint Class', 'Parent Property', 'Domain', 'Range',
'Inverse Property'
]
for id, row in terms.items():
for repl_label, repl_dict in replacements.items():
for col in axiom_cols:
if col not in row:
continue
row[col] = re.sub(repl_label, repl_dict['Replacement Label'], row[col])

base_writer = initialize_tsv(output_dir, 'cob-base.tsv', fieldnames, robot_row)
full_writer = initialize_tsv(output_dir, 'cob-full.tsv', fieldnames, robot_row)
root_writer = initialize_tsv(output_dir, 'cob-root.tsv', fieldnames, robot_row)

for id, row in terms.items():
module = row['COB Module'].strip()
if module == 'BASE':
base_writer.writerow(row)
if module not in ['', 'ROOT']:
full_writer.writerow(row)
if module != '':
root_writer.writerow(row)


def main():
parser = argparse.ArgumentParser(description='Split the main COB term template into multiple module templates')
parser.add_argument('input', type=str, help='The input COB term template')
parser.add_argument('outdir', type=str, help='The output directory')
args = parser.parse_args()

split(args.input, args.outdir)


if __name__ == '__main__':
main()

0 comments on commit 57f13da

Please sign in to comment.