Merge branch 'develop'

.github/workflows/ruby-unit-tests.yml

@@ -7,16 +7,22 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ['ruby', 'ruby-agraph'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend
     steps:
     - uses: actions/checkout@v3
     - name: copy config file
       run: cp config/config.test.rb config/config.rb
     - name: Build docker-compose
       run: docker-compose build
     - name: Run unit tests
+      # unit tests are run inside a container
+      # http://docs.codecov.io/docs/testing-with-docker
       run: |
         ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker-compose run $ci_env -e CI ruby bundle exec rake test TESTOPTS='-v'
+        docker-compose run $ci_env -e CI ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       with:

Dockerfile

@@ -14,7 +14,8 @@ COPY *.gemspec Gemfile* /srv/ontoportal/ncbo_annotator/
 
 WORKDIR /srv/ontoportal/ncbo_annotator
 
-RUN gem install bundler
+# set bundler to v2.4.22 which is the last version supported by ruby 2.7
+RUN gem install bundler -v 2.4.22
 ENV BUNDLE_PATH /bundle
 RUN bundle install
 COPY . /srv/ontoportal/ncbo_annotator

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/goo.git
-  revision: 657149d6b33813253fa7440252f69c04e0631190
+  revision: 75436fe8e387febc53e34ee31ff0e6dd837a9d3f
   branch: master
   specs:
     goo (0.0.2)
@@ -15,7 +15,7 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/ontologies_linked_data.git
-  revision: 7783784f9d2ceada9be706cf6c084d272ae653e8
+  revision: ee0013f0ee23876076bff9d9258b46371ec3b248
   branch: master
   specs:
     ontologies_linked_data (0.0.1)
@@ -63,7 +63,7 @@ GEM
     cube-ruby (0.0.3)
     date (3.3.4)
     docile (1.4.0)
-    domain_name (0.6.20231109)
+    domain_name (0.6.20240107)
     faraday (1.10.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
@@ -95,7 +95,7 @@ GEM
       concurrent-ruby (~> 1.0)
     json (2.7.1)
     json_pure (2.7.1)
-    libxml-ruby (4.1.2)
+    libxml-ruby (5.0.2)
     logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
@@ -105,22 +105,22 @@ GEM
       net-pop
       net-smtp
     method_source (1.0.0)
-    mime-types (3.5.1)
+    mime-types (3.5.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.1205)
     mini_mime (1.1.5)
     minitest (4.7.5)
     multi_json (1.15.0)
     multipart-post (2.3.0)
     net-http-persistent (2.9.4)
-    net-imap (0.4.8)
+    net-imap (0.4.9.1)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.4.0)
+    net-smtp (0.4.0.1)
       net-protocol
     netrc (0.11.0)
     oj (3.16.3)
@@ -141,7 +141,7 @@ GEM
       addressable (>= 2.2)
     redis (5.0.8)
       redis-client (>= 0.17.0)
-    redis-client (0.19.0)
+    redis-client (0.19.1)
       connection_pool
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -192,4 +192,4 @@ DEPENDENCIES
   sparql-client!
 
 BUNDLED WITH
-   2.3.15
+   2.4.22

config/config.test.rb

@@ -1,24 +1,31 @@
 # frozen_string_literal: true
 
 #
-# This file is designed for used for unit testing
+# configuration for unit testing
 #
-# All the defaults are set in
-# https://github.com/ncbo/ontologies_linked_data/blob/master/lib/ontologies_linked_data/config/config.rb
-###
-GOO_HOST   = ENV.include?('GOO_HOST')   ? ENV['GOO_HOST']   : 'localhost'
-GOO_PORT   = ENV.include?('GOO_PORT')   ? ENV['GOO_PORT']   : 9000
-REDIS_HOST = ENV.include?('REDIS_HOST') ? ENV['REDIS_HOST'] : 'localhost'
-REDIS_PORT = ENV.include?('REDIS_PORT') ? ENV['REDIS_PORT'] : 6379
-SOLR_HOST  = ENV.include?('SOLR_HOST')  ? ENV['SOLR_HOST']  : 'localhost'
-MGREP_HOST = ENV.include?('MGREP_HOST') ? ENV['MGREP_HOST'] : 'localhost'
-MGREP_PORT = ENV.include?('MGREP_PORT') ? ENV['MGREP_PORT'] : 55556
+GOO_HOST         = ENV.include?('GOO_HOST')         ? ENV['GOO_HOST']         : 'localhost'
+GOO_PORT         = ENV.include?('GOO_PORT')         ? ENV['GOO_PORT']         : 9000
+GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store"
+GOO_PATH_QUERY   = ENV.include?("GOO_PATH_QUERY")   ? ENV["GOO_PATH_QUERY"]   : "/sparql/"
+GOO_PATH_DATA    = ENV.include?("GOO_PATH_DATA")    ? ENV["GOO_PATH_DATA"]    : "/data/"
+GOO_PATH_UPDATE  = ENV.include?("GOO_PATH_UPDATE")  ? ENV["GOO_PATH_UPDATE"]  : "/update/"
+REDIS_HOST       = ENV.include?('REDIS_HOST')       ? ENV['REDIS_HOST']       : 'localhost'
+REDIS_PORT       = ENV.include?('REDIS_PORT')       ? ENV['REDIS_PORT']       : 6379
+MGREP_HOST       = ENV.include?('MGREP_HOST')       ? ENV['MGREP_HOST']       : 'localhost'
+MGREP_PORT       = ENV.include?('MGREP_PORT')       ? ENV['MGREP_PORT']       : 55556
+SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1"
+SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1"
+
 
 LinkedData.config do |config|
+  config.goo_backend_name           = GOO_BACKEND_NAME.to_s
   config.goo_host                   = GOO_HOST.to_s
   config.goo_port                   = GOO_PORT.to_i
-  config.search_server_url          = "http://#{SOLR_HOST}:8983/solr/term_search_core1".to_s
-  config.property_search_server_url = "http://#{SOLR_HOST}:8983/solr/prop_search_core1".to_s
+  config.goo_path_query             = GOO_PATH_QUERY.to_s
+  config.goo_path_data              = GOO_PATH_DATA.to_s
+  config.goo_path_update            = GOO_PATH_UPDATE.to_s
+  config.search_server_url          = SOLR_TERM_SEARCH_URL.to_s
+  config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s
 end
 Annotator.config do |config|
   config.mgrep_host            = MGREP_HOST.to_s

docker-compose.yml

@@ -1,39 +1,89 @@
 version: '3.8'
 
-services:
-  ruby:
+x-app: &app
     build:
       context: .
       args:
         RUBY_VERSION: '2.7'
+    environment: &env
+      COVERAGE: 'true' # enable simplecov code coverage
+      REDIS_HOST: redis-ut
+      REDIS_PORT: 6379
+      MGREP_HOST: mgrep-ut
+      MGREP_PORT: 55556
+      SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1
+      SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1
     volumes:
-      #- bundle:/bundle
       - .:/srv/ontoportal/ncbo_annotator
+    stdin_open: true
+    tty: true
+    command: "bundle exec rake test TESTOPTS='-v'"
+    depends_on: &depends_on
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      mgrep-ut:
+        condition: service_healthy
+
+services:
+  ruby:
+    <<: *app
     environment:
-      - BUNDLE_PATH=/bundle
-      - GOO_BACKEND_NAME=4store
-      - GOO_PORT=9000
-      - GOO_HOST=4store-ut
-      - REDIS_HOST=redis-ut
-      - REDIS_PORT=6379
-      - SOLR_HOST=solr-ut
-      - MGREP_HOST=mgrep-ut
-      - MGREP_PORT=55556
+      <<: *env
+      GOO_BACKEND_NAME: 4store
+      GOO_PORT: 9000
+      GOO_HOST: 4store-ut
     depends_on:
-      - solr-ut
-      - redis-ut
-      - 4store-ut
-      - mgrep-ut
-    command: "bundle exec rake test TESTOPTS='-v'"
+      <<: *depends_on
+      4store-ut:
+        condition: service_started
+    profiles:
+      - 4store
+
+  ruby-agraph:
+    <<: *app
+    environment:
+      <<: *env
+      GOO_BACKEND_NAME: ag
+      GOO_PORT: 10035
+      GOO_HOST: agraph-ut
+      GOO_PATH_QUERY: /repositories/bioportal_test
+      GOO_PATH_DATA: /repositories/bioportal_test/statements
+      GOO_PATH_UPDATE: /repositories/bioportal_test/statements
+    depends_on:
+      <<: *depends_on
+      agraph-ut:
+        condition: service_healthy
+    profiles:
+      - agraph
 
   solr-ut:
     image: ontoportal/solr-ut:0.0.2
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"]
+      start_period: 3s
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   redis-ut:
     image: redis
+    healthcheck:
+      test: redis-cli ping
+      interval: 10s
+      timeout: 3s
+      retries: 10
 
   mgrep-ut:
     image: ontoportal/mgrep:0.0.2
+    platform: linux/amd64
+    healthcheck:
+      test: ["CMD", "nc", "-z", "-v", "localhost", "55556"]
+      start_period: 3s
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   4store-ut:
     image: bde2020/4store
@@ -42,6 +92,30 @@ services:
       bash -c "4s-backend-setup --segments 4 ontoportal_kb
       && 4s-backend ontoportal_kb
       && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
+    profiles:
+      - 4store
+
+  agraph-ut:
+    image: franzinc/agraph:v8.0.0
+    platform: linux/amd64
+    environment:
+      - AGRAPH_SUPER_USER=test
+      - AGRAPH_SUPER_PASSWORD=xyzzy
+    shm_size: 1g
+    command: >
+       bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start
+       ; agtool repos create bioportal_test
+       ; agtool users add anonymous
+       ; agtool users grant anonymous root:bioportal_test:rw
+       ; tail -f /agraph/data/agraph.log"
+    healthcheck:
+      test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"]
+      start_period: 30s
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    profiles:
+      - agraph
 
 volumes:
   bundle:

lib/ncbo_annotator.rb

@@ -725,7 +725,7 @@ def mappings_for_class_ids(class_ids)
       def hierarchy_query(class_ids)
         # mdorf, 12/14/2023: AllegroGraph throws a MalformedQuery exception
         # if an ID is not of the proper URI format
-        class_ids.select! { |id| id =~ URI::regexp }
+        class_ids.select! { |id| id =~ /\A#{URI::regexp}\z/ }
         filter_ids = class_ids.map { |id| "?id = <#{id}>" } .join " || "
         query = <<eos
 SELECT DISTINCT ?id ?parent ?graph WHERE { GRAPH ?graph { ?id <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?parent . }

run-unit-tests.sh

@@ -4,8 +4,7 @@
 # add config for unit testing
 [ -f config/config.rb ] || cp config/config.test.rb config/config.rb
 docker-compose build
-#docker-compose up --exit-code-from unit-test
 
 docker-compose run --rm ruby bundle exec rake test TESTOPTS='-v'
-#docker-compose down
-docker-compose kill
+#docker-compose run --rm ruby-agraph bundle exec rake test TESTOPTS='-v'
+docker-compose --profile agraph --profile 4store kill

test/test_annotator.rb

@@ -97,7 +97,7 @@ def test_generate_dictionary_file
     assert @@redis.exists?(Annotator::Models::NcboAnnotator::MGREP_DICTIONARY_REFRESH_TIMESTAMP)
     assert @@redis.exists?(Annotator::Models::NcboAnnotator::LAST_MGREP_RESTART_TIMESTAMP)
     refresh_timestamp = @@redis.get(Annotator::Models::NcboAnnotator::MGREP_DICTIONARY_REFRESH_TIMESTAMP)
-    assert refresh_timestamp > start_timestamp
+    assert_operator refresh_timestamp, :>, start_timestamp
   end
 
   def test_mallet_recognizer