diff --git a/Gemfile b/Gemfile index 5082ef4e..f27148ae 100644 --- a/Gemfile +++ b/Gemfile @@ -3,20 +3,20 @@ gem 'activesupport', '~> 5' # see https://github.com/ncbo/ontologies_api/issues/69 gem 'bigdecimal' # gem 'faraday', '~> 1.9' +gem 'google-protobuf', '3.25.3' +gem 'json-ld' gem 'json-schema', '~> 2.0' gem 'multi_json' gem 'oj' +gem 'parallel' gem 'parseconfig' gem 'rack' gem 'rake', '~> 10.0' +gem 'request_store' gem 'rexml' # Investigate why unicorn fails to start under ruby 3 without adding rexml gem to the Gemfile gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' -gem 'request_store' -gem 'parallel' -gem 'json-ld' -gem 'google-protobuf', '3.25.3' # Rack middleware gem 'ffi', '~> 1.16.3' @@ -38,29 +38,27 @@ gem 'cube-ruby', require: 'cube' gem 'newrelic_rpm', group: [:default, :deployment] # HTTP server +gem "faye-websocket", "~> 0.11.3" +gem "puma", "~> 6.4" gem 'unicorn' gem 'unicorn-worker-killer' + + # Templating gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'feature/add-triple-store-logging' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ontoportal-lirmm/ncbo_ontology_recommender.git', branch: 'development' -gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' -gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' group :development do - # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 - gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' - gem 'rubocop' -end - -group :deployment do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false gem 'capistrano', '~> 3', require: false @@ -68,6 +66,9 @@ group :deployment do gem 'capistrano-locally', require: false gem 'capistrano-rbenv', require: false gem 'ed25519', '>= 1.2', '< 2.0', require: false + gem "listen", "~> 3.9" + gem 'pry' + gem 'rubocop' end diff --git a/Gemfile.lock b/Gemfile.lock index e2a97501..3f06b72f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: f8ac7b00e8d8b46d1eea04de014175525c1cdd83 - branch: development + revision: 6594c7dd689d96d4085ea66deffb9528b418b74c + branch: feature/add-triple-store-logging specs: goo (0.0.2) addressable (~> 2.8) @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: 37a9573c11978869a867050f8ec75e048c8b9b2b + revision: d50c624868dec11cb0afcc88ba422d021c77926c branch: master specs: ncbo_cron (0.0.1) @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 312ef426eeaa461e88fa23124ea5fd531f4276ba + revision: 0aa6219c44143b94135e01c78eb94ad99a5e8b32 branch: development specs: ontologies_linked_data (0.0.1) @@ -77,7 +77,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/sparql-client.git - revision: 59251e59346c9a69a67c88552ba55a1244eec602 + revision: 85f01b8c87ad9e630b6a34881fc5a2135a37e4b3 branch: development specs: sparql-client (3.2.2) @@ -92,14 +92,6 @@ GIT rack-post-body-to-params (0.1.8) activesupport (>= 2.3) -GIT - remote: https://github.com/palexander/shotgun.git - revision: db198224aaab2e4cb9b049adccb30e387d88bc3b - branch: ncbo - specs: - shotgun (0.9) - rack (>= 1.0) - GEM remote: https://rubygems.org/ specs: @@ -117,9 +109,9 @@ GEM base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) - bigdecimal (3.1.8) + bigdecimal (3.1.9) builder (3.3.0) - capistrano (3.19.1) + capistrano (3.19.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -132,17 +124,18 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.0) crack (0.4.5) rexml cube-ruby (0.0.3) dante (0.2.0) - date (3.4.0) + date (3.4.1) declarative (0.0.20) docile (1.4.1) domain_name (0.6.20240107) ed25519 (1.3.0) + eventmachine (1.2.7) faraday (2.8.1) base64 faraday-net_http (>= 2.0, < 3.1) @@ -150,6 +143,9 @@ GEM faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) + faye-websocket (0.11.3) + eventmachine (>= 0.12.0) + websocket-driver (>= 0.5.1) ffi (1.16.3) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) @@ -164,12 +160,12 @@ GEM google-analytics-data (0.6.1) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.13.1) + google-analytics-data-v1beta (0.14.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) google-apis-analytics_v3 (0.16.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.15.1) + google-apis-core (0.16.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -188,7 +184,7 @@ GEM google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.16.0) + googleapis-common-protos-types (1.18.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.11.2) faraday (>= 1.0, < 3.a) @@ -203,27 +199,30 @@ GEM haml (5.2.2) temple (>= 0.8.0) tilt - hashdiff (1.1.1) + hashdiff (1.1.2) htmlentities (4.3.4) http-accept (1.7.0) - http-cookie (1.0.7) + http-cookie (1.0.8) domain_name (~> 0.5) httpclient (2.8.3) - i18n (1.14.6) + i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.7.6) + json (2.9.1) json-ld (3.0.2) multi_json (~> 1.12) rdf (>= 2.2.8, < 4.0) json-schema (2.8.1) addressable (>= 2.4) - jwt (2.9.3) + jwt (2.10.1) base64 kgio (2.11.4) language_server-protocol (3.17.0.3) libxml-ruby (5.0.3) link_header (0.0.8) - logger (1.6.1) + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + logger (1.6.5) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -235,19 +234,19 @@ GEM mime-types (3.6.0) logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.1001) + mime-types-data (3.2025.0107) mini_mime (1.1.5) - minitest (5.25.1) + minitest (5.25.4) minitest-hooks (1.5.2) minitest (> 5.3) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - mutex_m (0.2.0) - net-http-persistent (4.0.4) + mutex_m (0.3.0) + net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.4.17) + net-imap (0.4.18) date net-protocol net-pop (0.1.2) @@ -262,25 +261,28 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.15.0) - oj (3.16.7) + newrelic_rpm (9.16.1) + nio4r (2.7.4) + oj (3.16.9) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) logger os (1.1.4) - ostruct (0.6.0) + ostruct (0.6.1) parallel (1.26.3) parseconfig (1.1.2) - parser (3.3.5.1) + parser (3.3.7.0) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) - pry (0.14.2) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.1.1) + puma (6.5.0) + nio4r (~> 2.0) racc (1.8.1) rack (1.6.13) rack-accept (0.4.5) @@ -295,12 +297,15 @@ GEM rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.1.0) + rack-test (2.2.0) rack (>= 1.3) rack-timeout (0.7.0) rainbow (3.1.1) raindrops (0.20.1) rake (10.5.0) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) rdf (3.2.11) link_header (~> 0.0, >= 0.0.8) rdf-raptor (3.2.0) @@ -319,14 +324,14 @@ GEM redcarpet (3.6.0) redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.22.2) + redis-client (0.23.2) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) redis-store (1.11.0) redis (>= 4, < 6) - regexp_parser (2.9.2) + regexp_parser (2.10.0) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -339,26 +344,26 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.3.9) + rexml (3.4.0) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.68.0) + rubocop (1.70.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rubocop-ast (>= 1.32.2, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.34.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.37.0) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) - rubyzip (2.3.2) + rubyzip (2.4.1) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) signet (0.19.0) @@ -397,13 +402,15 @@ GEM systemu (2.6.5) temple (0.10.3) thread_safe (0.3.6) - tilt (2.4.0) - timeout (0.4.1) + tilt (2.6.0) + timeout (0.4.3) trailblazer-option (0.1.2) tzinfo (1.2.11) thread_safe (~> 0.1) uber (0.1.0) - unicode-display_width (2.6.0) + unicode-display_width (3.1.4) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) @@ -416,7 +423,11 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.0) + webrick (1.9.1) + websocket-driver (0.7.7) + base64 + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) PLATFORMS x86_64-linux @@ -432,12 +443,14 @@ DEPENDENCIES crack (= 0.4.5) cube-ruby ed25519 (>= 1.2, < 2.0) + faye-websocket (~> 0.11.3) ffi (~> 1.16.3) goo! google-protobuf (= 3.25.3) haml (~> 5.2.2) json-ld json-schema (~> 2.0) + listen (~> 3.9) minitest (~> 5.0) minitest-hooks (~> 1.5) minitest-stub_any_instance @@ -450,6 +463,8 @@ DEPENDENCIES ontologies_linked_data! parallel parseconfig + pry + puma (~> 6.4) rack rack-accept (~> 0.4) rack-attack (~> 6.6.1) @@ -467,7 +482,6 @@ DEPENDENCIES request_store rexml rubocop - shotgun! simplecov simplecov-cobertura sinatra (~> 1.0) diff --git a/app.rb b/app.rb index e09178bd..48febbbb 100644 --- a/app.rb +++ b/app.rb @@ -185,3 +185,16 @@ Pry.start binding, :quiet => true exit end + + +require_relative 'lib/websockets/websockets_server' + +configure do + set :websocket_server, WebSocketServer.new(lambda { |env| + [200, { 'Content-Type' => 'text/plain' }, ['Hello World']] + }) +end + +Faye::WebSocket.load_adapter('puma') + + diff --git a/config/puma.rb b/config/puma.rb new file mode 100644 index 00000000..e44b324a --- /dev/null +++ b/config/puma.rb @@ -0,0 +1,21 @@ +workers Integer(ENV['WEB_CONCURRENCY'] || 0) +threads_count = Integer(ENV['MAX_THREADS'] || 5) +threads threads_count, threads_count + +pid = Process.pid +env = ENV['RACK_ENV'] || ENV['ENVIRONMENT'] || 'production' + +environment env +port 9393 + +if env == 'development' + puts "Reload enabled" + # Use the `listen` gem to reload the application when files change + require 'listen' + + listener = Listen.to('.', only: /\.rb$/) do |modified, added, removed| + # Reload the application when Ruby files change + Process.kill('SIGUSR1', pid) + end + listener.start +end \ No newline at end of file diff --git a/controllers/upload_ontology_controller.rb b/controllers/upload_ontology_controller.rb new file mode 100644 index 00000000..b7122ab4 --- /dev/null +++ b/controllers/upload_ontology_controller.rb @@ -0,0 +1,130 @@ +# app.rb +require 'sinatra/base' +require 'faye/websocket' +require 'thread' + +class UploadOntologyController < ApplicationController + require 'logger' + require 'observer' + + class ObservableLogger + include Observable + + def initialize(logfile) + @logger = Logger.new(logfile) + end + + def log(severity, message) + @logger.send(severity, message) + changed + notify_observers(severity, message) + end + + def flush + @logger.flush + end + + def info(message) + log(:info, message) + end + + def warn(message) + log(:warn, message) + end + + def error(message) + log(:error, message) + end + + def fatal(message) + log(:fatal, message) + end + end + + get '/ontologies/upload/ws' do + upload_ontology_ws_subscribe + end + + post '/ontologies/upload' do + ont, sub = perform_step(:ontology_creation, + "Start creating an ontology", + "Ontology created", + "Ontology creation failed" + ) do + create_ontology_from_file('SKOS') + end + + Thread.new do + # actions = { + # :process_rdf => true, + # :generate_labels => true, + # :extract_metadata => true, + # :index_search => true, + # :index_properties => true, + # :run_metrics => true, + # :process_annotator => true, + # :diff => true, + # :remote_pull => false + # } + break if ont.nil? + + begin + + steps = { + :process_rdf => { + label: 'Parse and save in RDF Store', + options: { process_rdf: true, extract_metadata: false, generate_labels: true } + }, + :extract_metadata => { + label: "Extract metadata", + options: { extract_metadata: true } + }, + # :index_search => { + # label: "Indexing terms", + # options: { index_search: true }, + # }, + :index_properties => { + label: "Indexing properties", + options: { index_properties: true }, + }, + :run_metrics => { + label: "Computing metrics", + options: { run_metrics: true } + } + } + + steps.each do |step, options| + perform_step(step, + "Start #{options[:label]}", + "#{options[:label]} ended successfully", + "#{options[:label]} failed") do + logger = ObservableLogger.new('app.log') + + observer = lambda do |severity, message| + broadcast_update(step, 1, message) + end + + logger.add_observer(observer, :call) + sub.process_submission(logger, options[:options]) + end + sub = LinkedData::Models::OntologySubmission.find(sub.id).first + sub.bring_remaining + end + + if sub.ready? + broadcast_update(:result, 2, "#{ont.acronym} parsed successfully") + broadcast_update(:result, 2, sub.metrics.to_hash.to_json) + end + rescue StandardError => e + broadcast_update(:result, 0, "#{ont.acronym} processing error: #{e.message}") + ensure + ont.delete + upload_ontology_ws_unsubscribe + end + end + status 200 + end + +end + + diff --git a/helpers/upload_ontology_helper.rb b/helpers/upload_ontology_helper.rb new file mode 100644 index 00000000..fd725b3f --- /dev/null +++ b/helpers/upload_ontology_helper.rb @@ -0,0 +1,68 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module UploadOntologyHelper + + def create_ontology_from_file(format = nil ) + LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: false, + acronym: 'INRAETHES', + name: 'INRAETHES', + ont_count: 1, + submission_count: 1, + ontology_format: format || 'OWL' + }) + ont = Ontology.find('INRAETHES-0').include(:acronym).first + sub = ont.latest_submission(status: :any) + add_file_to_submission(ont, sub) + [ont, sub] + end + + def perform_step(step, start_msg, end_msg, error_msg, &block) + broadcast_update(step, 1, start_msg) + outputs = nil + + time = Benchmark.realtime do + begin + outputs = block.call + rescue StandardError => e + outputs = nil + end + end + + if outputs + broadcast_update(step, 2, end_msg, time) + else + broadcast_update(step, 0, error_msg, time) + end + outputs + end + + def broadcast_update(id, status, message, time = nil) + msg = { + id: id, + status: status, + time: time, + message: message + } + upload_ontology_ws_broadcast(msg.to_json) + end + + def upload_ontology_ws_subscribe(channel_id = "",env = request.env) + websockets.subscribe("UploadOntologyController:#{channel_id}", env) + end + + def upload_ontology_ws_unsubscribe(channel_id = "") + websockets.unsubscribe("UploadOntologyController:#{channel_id}") + end + + def upload_ontology_ws_broadcast(channel_id = "", message) + websockets.broadcast("UploadOntologyController:#{channel_id}", message) + end + + end + end +end + +helpers Sinatra::Helpers::UploadOntologyHelper \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 00000000..0ebcec82 --- /dev/null +++ b/index.html @@ -0,0 +1,135 @@ + + + + + + Real-time File Processing + + + + + + + +
+

Real-time File Processing

+
+
+ + +
+ +
+
+
+ + + + + + + diff --git a/lib/websockets/websockets_server.rb b/lib/websockets/websockets_server.rb new file mode 100644 index 00000000..9c2dfad7 --- /dev/null +++ b/lib/websockets/websockets_server.rb @@ -0,0 +1,44 @@ +require 'faye/websocket' +require 'json' + +class WebSocketServer + + def initialize(app) + @app = app + @clients = Hash.new { |h, k| h[k] = [] } + end + + def subscribe(channel, env) + if Faye::WebSocket.websocket?(env) + ws = Faye::WebSocket.new(env) + + ws.on :open do |event| + puts 'WebSocket connection opened' + @clients[channel] ||= [] + @clients[channel] << ws + end + + ws.on :message do |event| + end + + ws.on :close do |event| + puts 'WebSocket connection closed' + @clients.delete(channel) + ws = nil + end + + ws.rack_response + else + @app.call(env) + end + end + + def unsubscribe(channel) + @clients[channel].each { |socket| socket.close} + end + + def broadcast(channel, message) + @clients[channel].each { |socket| socket.send(message)} + end + +end