From 16b660667fadad5927dccd60b9b9cfad8cb4adc3 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 18:33:04 +0800 Subject: [PATCH 01/12] Fixes --- README.md | 50 +++++++- docker-compose.master-replica.yml | 47 +++++++ lib/solr.rb | 10 ++ lib/solr/admin/collection_service.rb | 95 ++++++++++++++ lib/solr/admin/core_service.rb | 117 ++++++++++++++++++ lib/solr/configuration.rb | 16 ++- lib/solr/core_configuration/core_config.rb | 11 +- lib/solr/master_slave/configuration.rb | 11 +- lib/solr/support/url_helper.rb | 27 +++- spec/solr/admin/core_service_spec.rb | 105 ++++++++++++++++ .../cloud/collections_state_manager_spec.rb | 0 .../cloud/zookeeper_connection_spec.rb | 0 spec/{ => solr}/commands/commit_spec.rb | 0 spec/{ => solr}/commands/deleting_spec.rb | 0 spec/{ => solr}/commands/indexing_spec.rb | 0 spec/{ => solr}/configuration_spec.rb | 20 ++- spec/{ => solr}/indexing/document_spec.rb | 0 .../master_slave/configuration_spec.rb | 0 .../nodes_gray_list/in_memory_spec.rb | 0 .../query/request/and_filter_spec.rb | 0 .../dictionary_boost_function_spec.rb | 0 .../boosting/exists_boost_function_spec.rb | 0 ...eld_value_less_than_boost_function_spec.rb | 0 .../request/boosting/geodist_function_spec.rb | 0 .../boosting/ln_function_boost_spec.rb | 0 ...c_field_value_match_boost_function_spec.rb | 0 .../boosting/phrase_proximity_boost_spec.rb | 0 .../recent_field_value_boost_function_spec.rb | 0 .../boosting/scale_function_boost_spec.rb | 0 ...l_field_value_match_boost_function_spec.rb | 0 .../query/request/edismax_adapter_spec.rb | 0 spec/{ => solr}/query/request/facet_spec.rb | 0 .../query/request/field_list_spec.rb | 0 spec/{ => solr}/query/request/filter_spec.rb | 0 spec/{ => solr}/query/request/geofilt_spec.rb | 0 .../query/request/or_filter_spec.rb | 0 .../query/request/query_field_spec.rb | 0 spec/{ => solr}/query/request_spec.rb | 0 spec/{ => solr}/request/runner_spec.rb | 3 +- spec/{ => solr}/response_parser_spec.rb | 0 spec/{ => solr}/schema_helper_spec.rb | 0 spec/{ => solr}/solrb_spec.rb | 2 +- spec/{ => solr}/spatial_point_spec.rb | 0 spec/{ => solr}/spatial_rectangle_spec.rb | 0 .../support/string_extensions_spec.rb | 0 spec/solr/support/url_helper_spec.rb | 68 ++++++++++ spec/{ => solr}/update/request_spec.rb | 0 47 files changed, 558 insertions(+), 24 deletions(-) create mode 100644 docker-compose.master-replica.yml create mode 100644 lib/solr/admin/collection_service.rb create mode 100644 lib/solr/admin/core_service.rb create mode 100644 spec/solr/admin/core_service_spec.rb rename spec/{ => solr}/cloud/collections_state_manager_spec.rb (100%) rename spec/{ => solr}/cloud/zookeeper_connection_spec.rb (100%) rename spec/{ => solr}/commands/commit_spec.rb (100%) rename spec/{ => solr}/commands/deleting_spec.rb (100%) rename spec/{ => solr}/commands/indexing_spec.rb (100%) rename spec/{ => solr}/configuration_spec.rb (90%) rename spec/{ => solr}/indexing/document_spec.rb (100%) rename spec/{ => solr}/master_slave/configuration_spec.rb (100%) rename spec/{ => solr}/master_slave/nodes_gray_list/in_memory_spec.rb (100%) rename spec/{ => solr}/query/request/and_filter_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/dictionary_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/exists_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/field_value_less_than_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/geodist_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/ln_function_boost_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/numeric_field_value_match_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/phrase_proximity_boost_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/recent_field_value_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/scale_function_boost_spec.rb (100%) rename spec/{ => solr}/query/request/boosting/textual_field_value_match_boost_function_spec.rb (100%) rename spec/{ => solr}/query/request/edismax_adapter_spec.rb (100%) rename spec/{ => solr}/query/request/facet_spec.rb (100%) rename spec/{ => solr}/query/request/field_list_spec.rb (100%) rename spec/{ => solr}/query/request/filter_spec.rb (100%) rename spec/{ => solr}/query/request/geofilt_spec.rb (100%) rename spec/{ => solr}/query/request/or_filter_spec.rb (100%) rename spec/{ => solr}/query/request/query_field_spec.rb (100%) rename spec/{ => solr}/query/request_spec.rb (100%) rename spec/{ => solr}/request/runner_spec.rb (95%) rename spec/{ => solr}/response_parser_spec.rb (100%) rename spec/{ => solr}/schema_helper_spec.rb (100%) rename spec/{ => solr}/solrb_spec.rb (90%) rename spec/{ => solr}/spatial_point_spec.rb (100%) rename spec/{ => solr}/spatial_rectangle_spec.rb (100%) rename spec/{ => solr}/support/string_extensions_spec.rb (100%) create mode 100644 spec/solr/support/url_helper_spec.rb rename spec/{ => solr}/update/request_spec.rb (100%) diff --git a/README.md b/README.md index 6505ece..d4c5e4b 100644 --- a/README.md +++ b/README.md @@ -173,8 +173,8 @@ In solr master-slave mode you don't need to provide a solr url (`config.url` or ```ruby Solr.configure do |config| - config.master_url = 'localhost:8983' - config.slave_url = 'localhost:8984' + config.master_url = 'localhost:8983' # or ENV['SOLR_MASTER_URL'] + config.slave_url = 'localhost:8984' # or ENV['SOLR_SLAVE_URL'] # Disable select queries from master: config.disable_read_from_master = true # Specify Gray-list service @@ -553,6 +553,52 @@ SOLR_URL=http://localhost:8983/solr/test-core rspec docker-compose -f docker-compose.single.yml down -v ``` +### Master-Replica Setup + +```sh +# Start Solr master and replica nodes +docker-compose -f docker-compose.master-replica.yml up -d + +# Wait for both nodes to be healthy +docker-compose -f docker-compose.master-replica.yml ps + +# Setup master node +# First copy the default configset to the correct location on master +docker exec -u 0 solrb-master-replica-solr-master-1 sh -c "mkdir -p /var/solr/data/configsets && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ && chown -R solr:solr /var/solr/data/configsets" + +# Create test core on master +curl 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + +# Setup replica node +# Copy the default configset to the correct location on replica +docker exec -u 0 solrb-master-replica-solr-replica-1 sh -c "mkdir -p /var/solr/data/configsets && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ && chown -R solr:solr /var/solr/data/configsets" + +# Create test core on replica +curl 'http://localhost:8984/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + +# Enable replication on master +curl 'http://localhost:8983/solr/test-core/replication?command=enablereplication&master=true' + +# Enable replication on replica +curl 'http://localhost:8984/solr/test-core/replication?command=enablereplication&slave=true&masterUrl=http://solr-master:8983/solr/test-core' + +# Verify replication status on master +curl 'http://localhost:8983/solr/test-core/replication?command=details' + +# Verify replication status on replica +curl 'http://localhost:8984/solr/test-core/replication?command=details' + +# Disable field guessing on both nodes +curl http://localhost:8983/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' +curl http://localhost:8984/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' + +# Run specs with master-slave configuration +SOLR_MASTER_URL=http://localhost:8983/solr/test-core SOLR_SLAVE_URL=http://localhost:8984/solr/test-core rspec + +# Clean up +docker-compose -f docker-compose.master-replica.yml down -v +``` + ## Manual Setup with Docker Commands If you prefer more control or need to debug the setup, you can use the manual Docker commands: diff --git a/docker-compose.master-replica.yml b/docker-compose.master-replica.yml new file mode 100644 index 0000000..bc04604 --- /dev/null +++ b/docker-compose.master-replica.yml @@ -0,0 +1,47 @@ +name: solrb-master-replica + +services: + solr-master: + image: solr:9.7.0-slim + ports: + - "8983:8983" + volumes: + - solr_master_data:/var/solr + environment: + - SOLR_OPTS=-Denable.master=true + networks: + - solr_network + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8983/solr/"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + + solr-replica: + image: solr:9.7.0-slim + ports: + - "8984:8983" + volumes: + - solr_replica_data:/var/solr + environment: + - SOLR_OPTS=-Denable.replica=true -Dmaster.url=http://solr-master:8983/solr/test-core + networks: + - solr_network + depends_on: + solr-master: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8983/solr/"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + +networks: + solr_network: + driver: bridge + +volumes: + solr_master_data: + solr_replica_data: diff --git a/lib/solr.rb b/lib/solr.rb index 05263df..220ac9b 100644 --- a/lib/solr.rb +++ b/lib/solr.rb @@ -20,6 +20,8 @@ require 'solr/master_slave/helper_methods' require 'solr/helper_methods' require 'solr/commands' +require 'solr/admin/core_service' +require 'solr/admin/collection_service' module Solr class << self @@ -28,6 +30,14 @@ class << self include Solr::MasterSlave::HelperMethods include Solr::HelperMethods + def cores + @cores ||= Solr::Admin::CoreService.new + end + + def collections + @collections ||= Solr::Admin::CollectionService.new + end + CURRENT_CORE_CONFIG_VARIABLE_NAME = :solrb_current_core_config SOLR_NODE_URL_OVERRIDE_CONFIG = :solrb_node_url_override_config diff --git a/lib/solr/admin/collection_service.rb b/lib/solr/admin/collection_service.rb new file mode 100644 index 0000000..b5c0ccc --- /dev/null +++ b/lib/solr/admin/collection_service.rb @@ -0,0 +1,95 @@ +require 'solr/errors/solr_query_error' + +module Solr + module Admin + class CollectionService + def create(name:, config_set: '_default', num_shards: 1, replication_factor: 1, max_shards_per_node: nil) + params = { + action: 'CREATE', + name: name, + 'collection.configName': config_set, + numShards: num_shards, + replicationFactor: replication_factor + } + + params[:maxShardsPerNode] = max_shards_per_node if max_shards_per_node + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_collections, + path: 'admin/collections', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def delete(name:) + params = { + action: 'DELETE', + name: name + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_collections, + path: 'admin/collections', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def reload(name:) + params = { + action: 'RELOAD', + name: name + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_collections, + path: 'admin/collections', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def list + params = { + action: 'LIST' + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_collections, + path: 'admin/collections', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def exists?(name:) + response = list + response['collections'].include?(name.to_s) + rescue StandardError + false + end + + private + + def handle_response(response) + return JSON.parse(response.body) if response.status == 200 + + error = JSON.parse(response.body) rescue nil + if error && error['error'] + raise Solr::Errors::SolrQueryError, error['error']['msg'] + else + raise Solr::Errors::SolrQueryError, "Request failed with status #{response.status}" + end + end + end + end +end diff --git a/lib/solr/admin/core_service.rb b/lib/solr/admin/core_service.rb new file mode 100644 index 0000000..2371f9f --- /dev/null +++ b/lib/solr/admin/core_service.rb @@ -0,0 +1,117 @@ +require 'solr/errors/solr_query_error' + +module Solr + module Admin + class CoreService + def create(name:, config_set: '_default', config_dir: nil, schema_file: nil, data_dir: nil) + params = { + action: 'CREATE', + name: name, + configSet: config_set + } + + params[:configDir] = config_dir if config_dir + params[:schema] = schema_file if schema_file + params[:dataDir] = data_dir if data_dir + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_cores, + path: 'admin/cores', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + # Unloads a core from Solr. To completely remove a core, use unload with delete options. + # @param name [String] The name of the core to unload + # @param delete_index [Boolean] If true, will remove the index when unloading + # @param delete_data_dir [Boolean] If true, removes the data directory and all sub-directories + # @param delete_instance_dir [Boolean] If true, removes everything related to the core + def unload(name:, delete_index: false, delete_data_dir: false, delete_instance_dir: false) + params = { + action: 'UNLOAD', + core: name, + deleteIndex: delete_index, + deleteDataDir: delete_data_dir, + deleteInstanceDir: delete_instance_dir + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_cores, + path: 'admin/cores', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def reload(name:) + params = { + action: 'RELOAD', + core: name + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_cores, + path: 'admin/cores', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def rename(name:, new_name:) + params = { + action: 'RENAME', + core: name, + other: new_name + } + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_cores, + path: 'admin/cores', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def status(name: nil) + params = { action: 'STATUS' } + params[:core] = name if name + + url = Solr::Support::UrlHelper.build_request_url( + url: Solr::Support::UrlHelper.admin_base_url_for_cores, + path: 'admin/cores', + url_params: params + ) + + response = Solr::Connection.call(url: url, method: :get, body: nil) + handle_response(response) + end + + def exists?(name:) + response = status(name: name) + response.dig('status', name.to_s) != {} + end + + private + + def handle_response(response) + return JSON.parse(response.body) if response.status == 200 + + error = JSON.parse(response.body) rescue nil + if error && error['error'] + raise Solr::Errors::SolrQueryError, error['error']['msg'] + else + raise Solr::Errors::SolrQueryError, "Request failed with status #{response.status}" + end + end + end + end +end diff --git a/lib/solr/configuration.rb b/lib/solr/configuration.rb index 5884a55..38d1286 100644 --- a/lib/solr/configuration.rb +++ b/lib/solr/configuration.rb @@ -20,10 +20,10 @@ class Configuration attr_accessor :cores, :test_connection, :auth_user, :auth_password - attr_reader :url, :faraday_options, :faraday_configuration, :cloud_configuration, + attr_reader :url, :core, :faraday_options, :faraday_configuration, :cloud_configuration, :master_slave_configuration - attr_writer :url, :logger + attr_writer :url, :core, :logger def initialize @faraday_options = { @@ -72,12 +72,19 @@ def define_core(name: nil, default: false) end end + def core_name_from_solr_core_env + ENV['SOLR_CORE'] + end + def core_name_from_solr_url_env Solr::Support::UrlHelper.core_name_from_url(ENV['SOLR_URL']) end def build_env_url_core_config(name: nil) + name ||= core_name_from_solr_core_env name ||= core_name_from_solr_url_env + raise Solr::Errors::CouldNotInferImplicitCoreName if name.nil? || name == '' + Solr::CoreConfiguration::EnvUrlCoreConfig.new(name: name) end @@ -89,10 +96,7 @@ def validate_default_core_config!(default:) end def validate! - if !(url || - @cloud_configuration.zookeeper_url || - (@master_slave_configuration.master_url && @master_slave_configuration.slave_url) || - ENV['SOLR_URL']) + if !(url || ((@master_slave_configuration.master_url || ENV['SOLR_MASTER_URL']) && (@master_slave_configuration.slave_url || ENV['SOLR_SLAVE_URL'])) || ENV['SOLR_URL']) raise Solr::Errors::SolrUrlNotDefinedError end end diff --git a/lib/solr/core_configuration/core_config.rb b/lib/solr/core_configuration/core_config.rb index 9e45b9b..f7f6aae 100644 --- a/lib/solr/core_configuration/core_config.rb +++ b/lib/solr/core_configuration/core_config.rb @@ -4,7 +4,7 @@ class CoreConfig attr_reader :name, :fields def initialize(name:, fields:, default:) - @name = name + @name = name.nil? ? ENV['SOLR_CORE'] : name @fields = fields @default = default end @@ -36,8 +36,13 @@ def initialize(name: nil, fields: {}) end def url - raise ArgumentError, "Solr URL can't be nil" if ENV['SOLR_URL'].nil? - ENV['SOLR_URL'] + raise ArgumentError, "SOLR_URL can't be nil" if ENV['SOLR_URL'].nil? + + if ENV['SOLR_CORE'] && ENV['SOLR_CORE'] != '' + File.join(*[ENV['SOLR_URL'], name.to_s].compact) + else + ENV['SOLR_URL'] + end end end end diff --git a/lib/solr/master_slave/configuration.rb b/lib/solr/master_slave/configuration.rb index 671c903..5d1d6d5 100644 --- a/lib/solr/master_slave/configuration.rb +++ b/lib/solr/master_slave/configuration.rb @@ -4,9 +4,10 @@ module Solr module MasterSlave class Configuration - attr_accessor :master_url, :slave_url, :disable_read_from_master + attr_accessor :disable_read_from_master attr_reader :master_slave_enabled + attr_writer :master_url, :slave_url attr_writer :nodes_gray_list def enable_master_slave!(_) @@ -17,6 +18,14 @@ def master_slave_enabled? @master_slave_enabled end + def master_url + @master_url || ENV['SOLR_MASTER_URL'] + end + + def slave_url + @slave_url || ENV['SOLR_SLAVE_URL'] + end + def active_nodes_for(collection:) urls = [] urls.push(master_url) unless disable_read_from_master diff --git a/lib/solr/support/url_helper.rb b/lib/solr/support/url_helper.rb index 1a66e2c..5e841c0 100644 --- a/lib/solr/support/url_helper.rb +++ b/lib/solr/support/url_helper.rb @@ -37,12 +37,7 @@ def current_core def core_name_from_url(url) full_solr_core_uri = URI.parse(url) core_name = full_solr_core_uri.path.gsub('/solr', '').delete('/') - - if !core_name || core_name == '' - raise Solr::Errors::CouldNotInferImplicitCoreName - end - - core_name + core_name if core_name && core_name != '' end def solr_endpoint_from_url(url) @@ -54,6 +49,26 @@ def solr_endpoint_from_url(url) solr_endpoint end + + def admin_base_url_for_cores + if Solr.cloud_enabled? + raise Solr::Errors::SolrQueryError, 'Core management is not supported in SolrCloud mode. Use collections API instead.' + elsif Solr.master_slave_enabled? + Solr.configuration.master_url + else + solr_endpoint_from_url(Solr.configuration.url || ENV['SOLR_URL']) + end + end + + def admin_base_url_for_collections + if !Solr.cloud_enabled? + raise Solr::Errors::SolrQueryError, 'Collection management is only supported in SolrCloud mode. Use cores API instead.' + end + + # In SolrCloud mode, we can use any active node + base_url = Solr.active_nodes_for(collection: nil).first + base_url.chomp('/') + end end end end diff --git a/spec/solr/admin/core_service_spec.rb b/spec/solr/admin/core_service_spec.rb new file mode 100644 index 0000000..c294cae --- /dev/null +++ b/spec/solr/admin/core_service_spec.rb @@ -0,0 +1,105 @@ +RSpec.describe Solr::Admin::CoreService do + let(:service) { described_class.new } + let(:core_name) { SecureRandom.uuid } + let(:config_set) { '_default' } + + describe '#create' do + after do + service.unload(name: core_name, delete_index: true, delete_data_dir: true) + end + + it 'creates a new core' do + response = service.create(name: core_name, config_set: config_set) + expect(response['responseHeader']['status']).to eq(0) + end + end + + describe '#unload' do + before do + service.create(name: core_name, config_set: config_set) + end + + it 'unloads an existing core' do + response = service.unload(name: core_name) + expect(response['responseHeader']['status']).to eq(0) + end + end + + describe '#reload' do + before do + service.create(name: core_name, config_set: config_set) + end + + after do + service.unload(name: core_name, delete_index: true, delete_data_dir: true) + end + + it 'reloads an existing core' do + response = service.reload(name: core_name) + expect(response['responseHeader']['status']).to eq(0) + end + end + + describe '#rename' do + let(:new_name) { 'renamed_core' } + + before do + service.create(name: core_name, config_set: config_set) + end + + after do + service.unload(name: new_name, delete_index: true, delete_data_dir: true) + end + + it 'renames an existing core' do + response = service.rename(name: core_name, new_name: new_name) + expect(response['responseHeader']['status']).to eq(0) + end + end + + describe '#status' do + context 'when core exists' do + before do + service.create(name: core_name, config_set: config_set) + end + + after do + service.unload(name: core_name, delete_index: true, delete_data_dir: true) + end + + it 'returns core status' do + response = service.status(name: core_name) + expect(response['status'][core_name]).to be_a(Hash) + end + end + + context 'when core does not exist' do + it 'returns empty status for the core' do + response = service.status(name: 'non_existent_core') + expect(response['status']['non_existent_core']).to eq({}) + end + end + end + + describe '#exists?' do + context 'when core exists' do + before do + service.create(name: core_name, config_set: config_set) + end + + after do + service.unload(name: core_name, delete_index: true, delete_data_dir: true) + end + + it 'returns true' do + expect(service.exists?(name: core_name)).to be true + end + end + + context 'when core does not exist' do + it 'returns false' do + expect(service.exists?(name: 'non_existent_core')).to be false + end + end + end +end diff --git a/spec/cloud/collections_state_manager_spec.rb b/spec/solr/cloud/collections_state_manager_spec.rb similarity index 100% rename from spec/cloud/collections_state_manager_spec.rb rename to spec/solr/cloud/collections_state_manager_spec.rb diff --git a/spec/cloud/zookeeper_connection_spec.rb b/spec/solr/cloud/zookeeper_connection_spec.rb similarity index 100% rename from spec/cloud/zookeeper_connection_spec.rb rename to spec/solr/cloud/zookeeper_connection_spec.rb diff --git a/spec/commands/commit_spec.rb b/spec/solr/commands/commit_spec.rb similarity index 100% rename from spec/commands/commit_spec.rb rename to spec/solr/commands/commit_spec.rb diff --git a/spec/commands/deleting_spec.rb b/spec/solr/commands/deleting_spec.rb similarity index 100% rename from spec/commands/deleting_spec.rb rename to spec/solr/commands/deleting_spec.rb diff --git a/spec/commands/indexing_spec.rb b/spec/solr/commands/indexing_spec.rb similarity index 100% rename from spec/commands/indexing_spec.rb rename to spec/solr/commands/indexing_spec.rb diff --git a/spec/configuration_spec.rb b/spec/solr/configuration_spec.rb similarity index 90% rename from spec/configuration_spec.rb rename to spec/solr/configuration_spec.rb index 6d05723..241bbf2 100644 --- a/spec/configuration_spec.rb +++ b/spec/solr/configuration_spec.rb @@ -13,6 +13,18 @@ end end + context 'set core variable' do + before do + Solr.configure do |config| + config.core = '123-core' + end + end + + it 'uses the set url and core name' do + expect(Solr.configuration.core).to eq('123-core') + end + end + context 'specify nil url' do it 'raises exception' do expect do @@ -51,7 +63,7 @@ it 'users the set faraday_options' do expect(Solr.configuration.faraday_options).to eq(expected_config) core = Solr.configuration.default_core_config - expect(core.url).to eq(ENV['SOLR_URL']) + expect(core.url).to eq(File.join(*[ENV['SOLR_URL'], ENV['SOLR_CORE']].compact)) end end @@ -81,9 +93,9 @@ end it 'gets the core name from ENV config' do - expect(Solr.configuration.cores.keys).to eq([nil]) + expect(Solr.configuration.cores.keys).to eq([ENV['SOLR_CORE']]) core = Solr.configuration.default_core_config - expect(core.url).to eq(ENV['SOLR_URL']) + expect(core.url).to eq(File.join(*[ENV['SOLR_URL'], ENV['SOLR_CORE']].compact)) end end @@ -186,7 +198,7 @@ f.field :model end end - end.to raise_error("A core with name '' has been already defined") + end.to raise_error("A core with name '#{ENV['SOLR_CORE']}' has been already defined") end end end diff --git a/spec/indexing/document_spec.rb b/spec/solr/indexing/document_spec.rb similarity index 100% rename from spec/indexing/document_spec.rb rename to spec/solr/indexing/document_spec.rb diff --git a/spec/master_slave/configuration_spec.rb b/spec/solr/master_slave/configuration_spec.rb similarity index 100% rename from spec/master_slave/configuration_spec.rb rename to spec/solr/master_slave/configuration_spec.rb diff --git a/spec/master_slave/nodes_gray_list/in_memory_spec.rb b/spec/solr/master_slave/nodes_gray_list/in_memory_spec.rb similarity index 100% rename from spec/master_slave/nodes_gray_list/in_memory_spec.rb rename to spec/solr/master_slave/nodes_gray_list/in_memory_spec.rb diff --git a/spec/query/request/and_filter_spec.rb b/spec/solr/query/request/and_filter_spec.rb similarity index 100% rename from spec/query/request/and_filter_spec.rb rename to spec/solr/query/request/and_filter_spec.rb diff --git a/spec/query/request/boosting/dictionary_boost_function_spec.rb b/spec/solr/query/request/boosting/dictionary_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/dictionary_boost_function_spec.rb rename to spec/solr/query/request/boosting/dictionary_boost_function_spec.rb diff --git a/spec/query/request/boosting/exists_boost_function_spec.rb b/spec/solr/query/request/boosting/exists_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/exists_boost_function_spec.rb rename to spec/solr/query/request/boosting/exists_boost_function_spec.rb diff --git a/spec/query/request/boosting/field_value_less_than_boost_function_spec.rb b/spec/solr/query/request/boosting/field_value_less_than_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/field_value_less_than_boost_function_spec.rb rename to spec/solr/query/request/boosting/field_value_less_than_boost_function_spec.rb diff --git a/spec/query/request/boosting/geodist_function_spec.rb b/spec/solr/query/request/boosting/geodist_function_spec.rb similarity index 100% rename from spec/query/request/boosting/geodist_function_spec.rb rename to spec/solr/query/request/boosting/geodist_function_spec.rb diff --git a/spec/query/request/boosting/ln_function_boost_spec.rb b/spec/solr/query/request/boosting/ln_function_boost_spec.rb similarity index 100% rename from spec/query/request/boosting/ln_function_boost_spec.rb rename to spec/solr/query/request/boosting/ln_function_boost_spec.rb diff --git a/spec/query/request/boosting/numeric_field_value_match_boost_function_spec.rb b/spec/solr/query/request/boosting/numeric_field_value_match_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/numeric_field_value_match_boost_function_spec.rb rename to spec/solr/query/request/boosting/numeric_field_value_match_boost_function_spec.rb diff --git a/spec/query/request/boosting/phrase_proximity_boost_spec.rb b/spec/solr/query/request/boosting/phrase_proximity_boost_spec.rb similarity index 100% rename from spec/query/request/boosting/phrase_proximity_boost_spec.rb rename to spec/solr/query/request/boosting/phrase_proximity_boost_spec.rb diff --git a/spec/query/request/boosting/recent_field_value_boost_function_spec.rb b/spec/solr/query/request/boosting/recent_field_value_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/recent_field_value_boost_function_spec.rb rename to spec/solr/query/request/boosting/recent_field_value_boost_function_spec.rb diff --git a/spec/query/request/boosting/scale_function_boost_spec.rb b/spec/solr/query/request/boosting/scale_function_boost_spec.rb similarity index 100% rename from spec/query/request/boosting/scale_function_boost_spec.rb rename to spec/solr/query/request/boosting/scale_function_boost_spec.rb diff --git a/spec/query/request/boosting/textual_field_value_match_boost_function_spec.rb b/spec/solr/query/request/boosting/textual_field_value_match_boost_function_spec.rb similarity index 100% rename from spec/query/request/boosting/textual_field_value_match_boost_function_spec.rb rename to spec/solr/query/request/boosting/textual_field_value_match_boost_function_spec.rb diff --git a/spec/query/request/edismax_adapter_spec.rb b/spec/solr/query/request/edismax_adapter_spec.rb similarity index 100% rename from spec/query/request/edismax_adapter_spec.rb rename to spec/solr/query/request/edismax_adapter_spec.rb diff --git a/spec/query/request/facet_spec.rb b/spec/solr/query/request/facet_spec.rb similarity index 100% rename from spec/query/request/facet_spec.rb rename to spec/solr/query/request/facet_spec.rb diff --git a/spec/query/request/field_list_spec.rb b/spec/solr/query/request/field_list_spec.rb similarity index 100% rename from spec/query/request/field_list_spec.rb rename to spec/solr/query/request/field_list_spec.rb diff --git a/spec/query/request/filter_spec.rb b/spec/solr/query/request/filter_spec.rb similarity index 100% rename from spec/query/request/filter_spec.rb rename to spec/solr/query/request/filter_spec.rb diff --git a/spec/query/request/geofilt_spec.rb b/spec/solr/query/request/geofilt_spec.rb similarity index 100% rename from spec/query/request/geofilt_spec.rb rename to spec/solr/query/request/geofilt_spec.rb diff --git a/spec/query/request/or_filter_spec.rb b/spec/solr/query/request/or_filter_spec.rb similarity index 100% rename from spec/query/request/or_filter_spec.rb rename to spec/solr/query/request/or_filter_spec.rb diff --git a/spec/query/request/query_field_spec.rb b/spec/solr/query/request/query_field_spec.rb similarity index 100% rename from spec/query/request/query_field_spec.rb rename to spec/solr/query/request/query_field_spec.rb diff --git a/spec/query/request_spec.rb b/spec/solr/query/request_spec.rb similarity index 100% rename from spec/query/request_spec.rb rename to spec/solr/query/request_spec.rb diff --git a/spec/request/runner_spec.rb b/spec/solr/request/runner_spec.rb similarity index 95% rename from spec/request/runner_spec.rb rename to spec/solr/request/runner_spec.rb index 99d3e18..85aef9b 100644 --- a/spec/request/runner_spec.rb +++ b/spec/solr/request/runner_spec.rb @@ -11,7 +11,8 @@ describe '.call' do it 'calls solr connection' do - expect(solr_connection).to receive(:call).with(url: "#{ENV['SOLR_URL']}/select", + url = File.join(*[ENV['SOLR_URL'], ENV['SOLR_CORE'], 'select'].compact) + expect(solr_connection).to receive(:call).with(url: url, method: :get, body: '{ "some_json_key": "some_json_value" }') subject.call diff --git a/spec/response_parser_spec.rb b/spec/solr/response_parser_spec.rb similarity index 100% rename from spec/response_parser_spec.rb rename to spec/solr/response_parser_spec.rb diff --git a/spec/schema_helper_spec.rb b/spec/solr/schema_helper_spec.rb similarity index 100% rename from spec/schema_helper_spec.rb rename to spec/solr/schema_helper_spec.rb diff --git a/spec/solrb_spec.rb b/spec/solr/solrb_spec.rb similarity index 90% rename from spec/solrb_spec.rb rename to spec/solr/solrb_spec.rb index f6fe830..257fc73 100644 --- a/spec/solrb_spec.rb +++ b/spec/solr/solrb_spec.rb @@ -5,7 +5,7 @@ describe '.current_core_config' do it 'uses default url' do - expect(Solr.current_core_config.url).to eq(ENV['SOLR_URL']) + expect(Solr.current_core_config.url).to eq(File.join(*[ENV['SOLR_URL'], ENV['SOLR_CORE']].compact)) end end diff --git a/spec/spatial_point_spec.rb b/spec/solr/spatial_point_spec.rb similarity index 100% rename from spec/spatial_point_spec.rb rename to spec/solr/spatial_point_spec.rb diff --git a/spec/spatial_rectangle_spec.rb b/spec/solr/spatial_rectangle_spec.rb similarity index 100% rename from spec/spatial_rectangle_spec.rb rename to spec/solr/spatial_rectangle_spec.rb diff --git a/spec/support/string_extensions_spec.rb b/spec/solr/support/string_extensions_spec.rb similarity index 100% rename from spec/support/string_extensions_spec.rb rename to spec/solr/support/string_extensions_spec.rb diff --git a/spec/solr/support/url_helper_spec.rb b/spec/solr/support/url_helper_spec.rb new file mode 100644 index 0000000..9cac790 --- /dev/null +++ b/spec/solr/support/url_helper_spec.rb @@ -0,0 +1,68 @@ +RSpec.describe Solr::Support::UrlHelper do + describe '#solr_endpoint_from_url' do + subject { Solr::Support::UrlHelper.solr_endpoint_from_url(url) } + + context 'when url ends with /solr' do + let(:url) { 'http://localhost:8983/solr/core1' } + + it 'returns the solr endpoint' do + expect(subject).to eq('http://localhost:8983/solr') + end + end + + context 'when url contains /solr and core name' do + let(:url) { 'http://localhost:8983/solr/core1' } + + it 'returns the solr endpoint' do + expect(subject).to eq('http://localhost:8983/solr') + end + end + end + + describe '#admin_base_url_for_cores' do + context 'when SOLR_URL is set' do + before do + stub_const('ENV', { 'SOLR_URL' => 'http://localhost:8983/solr/core1' }) + end + + it 'returns the admin base URL for cores' do + expect(Solr::Support::UrlHelper.admin_base_url_for_cores).to eq('http://localhost:8983/solr') + end + end + + context 'when SOLR_URL and SOLR_CORE are set' do + before do + stub_const('ENV', { 'SOLR_URL' => 'http://localhost:8983/solr', 'SOLR_CORE' => 'core1' }) + end + + it 'returns the admin base URL for cores' do + expect(Solr::Support::UrlHelper.admin_base_url_for_cores).to eq('http://localhost:8983/solr') + end + end + + context 'when url is set in configuration' do + before do + Solr.configure do |config| + config.url = 'http://localhost:8983/solr/core1' + end + end + + it 'returns the admin base URL for cores' do + expect(Solr::Support::UrlHelper.admin_base_url_for_cores).to eq('http://localhost:8983/solr') + end + end + + context 'when url and core are set in configuration' do + before do + Solr.configure do |config| + config.url = 'http://localhost:8983/solr' + config.core = 'core1' + end + end + + it 'returns the admin base URL for cores' do + expect(Solr::Support::UrlHelper.admin_base_url_for_cores).to eq('http://localhost:8983/solr') + end + end + end +end diff --git a/spec/update/request_spec.rb b/spec/solr/update/request_spec.rb similarity index 100% rename from spec/update/request_spec.rb rename to spec/solr/update/request_spec.rb From 50c4eff2f3cb24afaf801aa2e9e19ebbab0f53fd Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 20:30:57 +0800 Subject: [PATCH 02/12] Add securerandom --- solrb.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solrb.gemspec b/solrb.gemspec index a6f2c88..7f090aa 100644 --- a/solrb.gemspec +++ b/solrb.gemspec @@ -30,5 +30,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rake', '~> 10.0' spec.add_development_dependency 'rspec', '~> 3.13.0' spec.add_development_dependency 'rubocop', '~> 1.71.0' - spec.add_development_dependency 'simplecov' + spec.add_development_dependency 'securerandom' + spec.add_development_dependency 'webmock' end From d83abd87e36da9c1d44880eb288b7a9c7cac431c Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 20:35:02 +0800 Subject: [PATCH 03/12] Require SecureRandom --- spec/spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 474bcef..90deba7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,6 +9,7 @@ require 'bundler/setup' require 'pry' +require 'securerandom' require 'solr' RSpec.configure do |config| From 9f1cea4642fc1d562bf69ad0e74e1c6600ead0a5 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 20:39:41 +0800 Subject: [PATCH 04/12] Update readme --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/README.md b/README.md index d4c5e4b..120a925 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ Object-Oriented approach to Solr in Ruby. * [Master-slave](#master-slave) * [Gray list](#gray-list) * [Basic Authentication](#basic-authentication) +* [Core Management](#core-management) + * [Creating a Core](#creating-a-core) + * [Managing Cores](#managing-cores) * [Indexing](#indexing) * [Querying](#querying) * [Simple Query](#simple-query) @@ -223,6 +226,54 @@ Solr.configure do |config| end ``` +# Core Management + +Solrb provides a comprehensive API for managing Solr cores through the `Solr.cores` interface. Here are the available operations: + +## Creating a Core + +```ruby +# Create a core with default configuration +Solr.cores.create(name: 'my-core') + +# Create a core with custom configuration +Solr.cores.create( + name: 'my-core', + config_set: '_default', # Optional, defaults to '_default' + config_dir: 'path/to/config', # Optional + schema_file: 'path/to/schema.xml', # Optional + data_dir: 'path/to/data' # Optional +) +``` + +## Managing Cores + +```ruby +# Check if a core exists +Solr.cores.exists?(name: 'my-core') + +# Get status of all cores or a specific core +Solr.cores.status # All cores +Solr.cores.status(name: 'my-core') # Specific core + +# Reload a core +Solr.cores.reload(name: 'my-core') + +# Rename a core +Solr.cores.rename(name: 'old-name', new_name: 'new-name') + +# Unload a core +Solr.cores.unload(name: 'my-core') + +# Unload and delete a core completely +Solr.cores.unload( + name: 'my-core', + delete_index: true, # Remove the index + delete_data_dir: true, # Remove the data directory + delete_instance_dir: true # Remove everything related to the core +) +``` + # Indexing ```ruby From 0bf18c18febb941c8e5dad82403780cb353815e1 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 20:55:33 +0800 Subject: [PATCH 05/12] Run solr in docker compose on CI --- .github/workflows/tests.yaml | 16 +++++++++------- docker-compose.single.yml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 42f7643..d4885ca 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -6,18 +6,13 @@ env: SOLR_URL: http://localhost:8983/solr/test-core jobs: - rspec: + rspec_single: name: Rspec on Ruby ${{ matrix.ruby }} with Solr ${{ matrix.solr }} runs-on: ubuntu-latest strategy: matrix: ruby: ['2.7', '3.0', '3.2'] solr: ['8.11.2', '9.7.0'] - services: - solr: - image: solr:${{ matrix.solr }} - ports: - - 8983:8983 steps: - uses: actions/checkout@v2 @@ -26,6 +21,13 @@ jobs: ruby-version: ${{ matrix.ruby }} bundler-cache: true + - name: Run docker-compose + uses: hoverkraft-tech/compose-action@v2 + env: + SOLR_VERSION: ${{ matrix.solr }} + with: + compose-file: docker-compose.single.yml + - name: Prepare default configset run: | container_id=$(docker ps -q) @@ -42,4 +44,4 @@ jobs: curl http://localhost:8983/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' - name: Rspec - run: bundle exec rspec --format progress + run: bundle exec rspec --format progress --tag ~installation:master-replica --tag ~installation:cloud diff --git a/docker-compose.single.yml b/docker-compose.single.yml index fb40a6a..3a230f5 100644 --- a/docker-compose.single.yml +++ b/docker-compose.single.yml @@ -2,7 +2,7 @@ version: '3.8' services: solr: - image: solr:9.7.0-slim + image: solr:${SOLR_VERSION:-9.7.0}-slim ports: - "8983:8983" volumes: From 8a89f278dd08bc32a00dbbce32add9ee60b3373a Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:00:40 +0800 Subject: [PATCH 06/12] Improve solr healthcheck --- docker-compose.single.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker-compose.single.yml b/docker-compose.single.yml index 3a230f5..6cca13f 100644 --- a/docker-compose.single.yml +++ b/docker-compose.single.yml @@ -8,11 +8,14 @@ services: volumes: - solr_single_data:/var/solr healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8983/solr/"] - interval: 5s + test: | + curl -sf 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q '"status":{' \ + && curl -sf 'http://localhost:8983/solr/admin/info/system' | grep -q '"status":"ok"' \ + && curl -sf 'http://localhost:8983/solr/admin/ping?wt=json' | grep -q '"status":"OK"' + interval: 10s timeout: 5s - retries: 3 - start_period: 5s + retries: 5 + start_period: 20s volumes: solr_single_data: From 4391f2cb080bb23122385f00bc0d2eda571458a1 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:09:58 +0800 Subject: [PATCH 07/12] Fixes --- .github/workflows/tests.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d4885ca..cc1cc97 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,10 +30,9 @@ jobs: - name: Prepare default configset run: | - container_id=$(docker ps -q) - docker exec -u 0 $container_id sh -c "mkdir /var/solr/data/configsets \ - && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ \ - && chown -R solr:solr /var/solr/data/configsets" + docker exec -u 0 solrb-solr-1 sh -c \ + "mkdir -p /var/solr/data/configsets && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ \ + && chown -R solr:solr /var/solr/data/configsets" - name: Create a test core run: | From 4daaec6454f261975591a4a6d691958297a5a3ec Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:11:47 +0800 Subject: [PATCH 08/12] Fixes --- .github/workflows/tests.yaml | 32 +++++++++++++++++++++++++++++++- docker-compose.single.yml | 5 +---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cc1cc97..6a84c6d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -28,15 +28,45 @@ jobs: with: compose-file: docker-compose.single.yml + - name: Wait for Solr to be ready + timeout-minutes: 2 + run: | + echo "Waiting for Solr to be ready..." + while ! curl -s 'http://localhost:8983/solr/admin/info/system' > /dev/null; do + sleep 2 + echo "Still waiting for Solr..." + done + echo "Solr is up!" + - name: Prepare default configset run: | docker exec -u 0 solrb-solr-1 sh -c \ "mkdir -p /var/solr/data/configsets && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ \ && chown -R solr:solr /var/solr/data/configsets" + - name: Wait for configset to be available + timeout-minutes: 1 + run: | + echo "Waiting for configset to be ready..." + while ! curl -s 'http://localhost:8983/solr/admin/configs?action=LIST' | grep -q '_default'; do + sleep 2 + echo "Still waiting for configset..." + done + echo "Configset is ready!" + - name: Create a test core run: | - curl 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + curl --retry 5 --retry-delay 2 -sS 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + + - name: Verify core creation + timeout-minutes: 1 + run: | + echo "Verifying core creation..." + while ! curl -s 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q '"test-core":'; do + sleep 2 + echo "Waiting for core to be ready..." + done + echo "Core is ready!" - name: Disable field type guessing run: | diff --git a/docker-compose.single.yml b/docker-compose.single.yml index 6cca13f..4f9765d 100644 --- a/docker-compose.single.yml +++ b/docker-compose.single.yml @@ -8,10 +8,7 @@ services: volumes: - solr_single_data:/var/solr healthcheck: - test: | - curl -sf 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q '"status":{' \ - && curl -sf 'http://localhost:8983/solr/admin/info/system' | grep -q '"status":"ok"' \ - && curl -sf 'http://localhost:8983/solr/admin/ping?wt=json' | grep -q '"status":"OK"' + test: curl --fail http://localhost:8983/solr/admin/info/system interval: 10s timeout: 5s retries: 5 From 5ebbdade1fcf45d882f0eda8b772f2ae5b530c13 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:14:06 +0800 Subject: [PATCH 09/12] Fixes --- .github/workflows/tests.yaml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6a84c6d..6303c45 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -40,19 +40,13 @@ jobs: - name: Prepare default configset run: | - docker exec -u 0 solrb-solr-1 sh -c \ - "mkdir -p /var/solr/data/configsets && cp -R /opt/solr/server/solr/configsets/_default /var/solr/data/configsets/ \ - && chown -R solr:solr /var/solr/data/configsets" - - - name: Wait for configset to be available - timeout-minutes: 1 - run: | - echo "Waiting for configset to be ready..." - while ! curl -s 'http://localhost:8983/solr/admin/configs?action=LIST' | grep -q '_default'; do - sleep 2 - echo "Still waiting for configset..." - done - echo "Configset is ready!" + container_id=$(docker ps -q --filter name=solrb-solr) + echo "Using container: $container_id" + docker exec -u 0 $container_id sh -c \ + "mkdir -p /var/solr/data/configsets/_default && \ + cp -R /opt/solr/server/solr/configsets/_default/* /var/solr/data/configsets/_default/ && \ + chown -R solr:solr /var/solr/data/configsets && \ + ls -la /var/solr/data/configsets/_default/" - name: Create a test core run: | From af1b54d0aaffd1085846f73414f9b2fde7e35027 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:19:52 +0800 Subject: [PATCH 10/12] Add master-replica job to CI --- .github/workflows/tests.yaml | 106 ++++++++++++++++++++++++++- docker-compose.master-replica.yml | 20 ++--- spec/solr/admin/core_service_spec.rb | 2 +- 3 files changed, 114 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6303c45..fa331d9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,9 +2,6 @@ name: Tests 'on': [push] -env: - SOLR_URL: http://localhost:8983/solr/test-core - jobs: rspec_single: name: Rspec on Ruby ${{ matrix.ruby }} with Solr ${{ matrix.solr }} @@ -13,6 +10,8 @@ jobs: matrix: ruby: ['2.7', '3.0', '3.2'] solr: ['8.11.2', '9.7.0'] + env: + SOLR_URL: http://localhost:8983/solr/test-core steps: - uses: actions/checkout@v2 @@ -68,3 +67,104 @@ jobs: - name: Rspec run: bundle exec rspec --format progress --tag ~installation:master-replica --tag ~installation:cloud + + rspec_master_replica: + name: Rspec Master-Replica on Ruby ${{ matrix.ruby }} with Solr ${{ matrix.solr }} + runs-on: ubuntu-latest + strategy: + matrix: + ruby: ['2.7', '3.0', '3.2'] + solr: ['8.11.2', '9.7.0'] + env: + SOLR_MASTER_URL: http://localhost:8983/solr/test-core + SOLR_REPLICA_URL: http://localhost:8984/solr/test-core + steps: + - uses: actions/checkout@v2 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + + - name: Run docker-compose + uses: hoverkraft-tech/compose-action@v2 + env: + SOLR_VERSION: ${{ matrix.solr }} + with: + compose-file: docker-compose.master-replica.yml + + - name: Wait for Solr master to be ready + timeout-minutes: 2 + run: | + echo "Waiting for Solr master to be ready..." + while ! curl -s 'http://localhost:8983/solr/admin/info/system' > /dev/null; do + sleep 2 + echo "Still waiting for Solr master..." + done + echo "Solr master is up!" + + - name: Wait for Solr replica to be ready + timeout-minutes: 2 + run: | + echo "Waiting for Solr replica to be ready..." + while ! curl -s 'http://localhost:8984/solr/admin/info/system' > /dev/null; do + sleep 2 + echo "Still waiting for Solr replica..." + done + echo "Solr replica is up!" + + - name: Prepare master configset + run: | + container_id=$(docker ps -q --filter name=solrb-master-replica-solr-master) + echo "Using master container: $container_id" + docker exec -u 0 $container_id sh -c \ + "mkdir -p /var/solr/data/configsets/_default && \ + cp -R /opt/solr/server/solr/configsets/_default/* /var/solr/data/configsets/_default/ && \ + chown -R solr:solr /var/solr/data/configsets && \ + ls -la /var/solr/data/configsets/_default/" + + - name: Prepare replica configset + run: | + container_id=$(docker ps -q --filter name=solrb-master-replica-solr-replica) + echo "Using replica container: $container_id" + docker exec -u 0 $container_id sh -c \ + "mkdir -p /var/solr/data/configsets/_default && \ + cp -R /opt/solr/server/solr/configsets/_default/* /var/solr/data/configsets/_default/ && \ + chown -R solr:solr /var/solr/data/configsets && \ + ls -la /var/solr/data/configsets/_default/" + + - name: Create master core + run: | + curl --retry 5 --retry-delay 2 -sS 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + + - name: Create replica core + run: | + curl --retry 5 --retry-delay 2 -sS 'http://localhost:8984/solr/admin/cores?action=CREATE&name=test-core&configSet=_default' + + - name: Verify cores creation + timeout-minutes: 1 + run: | + echo "Verifying master core..." + while ! curl -s 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q '"test-core":'; do + sleep 2 + echo "Waiting for master core..." + done + echo "Master core is ready!" + + echo "Verifying replica core..." + while ! curl -s 'http://localhost:8984/solr/admin/cores?action=STATUS' | grep -q '"test-core":'; do + sleep 2 + echo "Waiting for replica core..." + done + echo "Replica core is ready!" + + - name: Disable field type guessing on master + run: | + curl http://localhost:8983/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' + + - name: Disable field type guessing on replica + run: | + curl http://localhost:8984/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' + + - name: Rspec + run: bundle exec rspec --format progress --tag installation:master-replica --tag ~installation:cloud --tag ~installation:single diff --git a/docker-compose.master-replica.yml b/docker-compose.master-replica.yml index bc04604..8cbfb46 100644 --- a/docker-compose.master-replica.yml +++ b/docker-compose.master-replica.yml @@ -2,7 +2,7 @@ name: solrb-master-replica services: solr-master: - image: solr:9.7.0-slim + image: solr:${SOLR_VERSION:-9.7.0}-slim ports: - "8983:8983" volumes: @@ -12,14 +12,14 @@ services: networks: - solr_network healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8983/solr/"] - interval: 5s + test: curl --fail http://localhost:8983/solr/admin/info/system + interval: 10s timeout: 5s - retries: 3 - start_period: 5s + retries: 5 + start_period: 20s solr-replica: - image: solr:9.7.0-slim + image: solr:${SOLR_VERSION:-9.7.0}-slim ports: - "8984:8983" volumes: @@ -32,11 +32,11 @@ services: solr-master: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8983/solr/"] - interval: 5s + test: curl --fail http://localhost:8983/solr/admin/info/system + interval: 10s timeout: 5s - retries: 3 - start_period: 5s + retries: 5 + start_period: 20s networks: solr_network: diff --git a/spec/solr/admin/core_service_spec.rb b/spec/solr/admin/core_service_spec.rb index c294cae..494d498 100644 --- a/spec/solr/admin/core_service_spec.rb +++ b/spec/solr/admin/core_service_spec.rb @@ -1,4 +1,4 @@ -RSpec.describe Solr::Admin::CoreService do +RSpec.describe Solr::Admin::CoreService, installation: :single do let(:service) { described_class.new } let(:core_name) { SecureRandom.uuid } let(:config_set) { '_default' } From 3f78e4870534a0f4cea6a314be9609f0e389c3b8 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Wed, 5 Feb 2025 21:20:20 +0800 Subject: [PATCH 11/12] Fixes --- .github/workflows/tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index fa331d9..ce4f241 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,8 +8,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby: ['2.7', '3.0', '3.2'] - solr: ['8.11.2', '9.7.0'] + ruby: ['3.2'] + solr: ['9.7.0'] env: SOLR_URL: http://localhost:8983/solr/test-core steps: @@ -73,8 +73,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby: ['2.7', '3.0', '3.2'] - solr: ['8.11.2', '9.7.0'] + ruby: ['3.2'] + solr: ['9.7.0'] env: SOLR_MASTER_URL: http://localhost:8983/solr/test-core SOLR_REPLICA_URL: http://localhost:8984/solr/test-core From 7ae276888b11a816e975215bd9fbc2bfb3e8a1e5 Mon Sep 17 00:00:00 2001 From: Vladislav Syabruk Date: Thu, 6 Feb 2025 15:29:20 +0800 Subject: [PATCH 12/12] Fixes --- .github/workflows/tests.yaml | 2 +- lib/solr/configuration.rb | 8 +++++++- lib/solr/core_configuration/core_config.rb | 13 +++++++++---- lib/solr/support/url_helper.rb | 2 +- spec/solr/configuration_spec.rb | 2 +- spec/solr/solrb_spec.rb | 6 +++++- spec/spec_helper.rb | 22 ++++++++++++++++++++-- 7 files changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ce4f241..80217ef 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -167,4 +167,4 @@ jobs: curl http://localhost:8984/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}' - name: Rspec - run: bundle exec rspec --format progress --tag installation:master-replica --tag ~installation:cloud --tag ~installation:single + run: bundle exec rspec --format progress --tag installation:master_replica --tag ~installation:cloud --tag ~installation:single diff --git a/lib/solr/configuration.rb b/lib/solr/configuration.rb index 38d1286..b06fbc8 100644 --- a/lib/solr/configuration.rb +++ b/lib/solr/configuration.rb @@ -77,7 +77,13 @@ def core_name_from_solr_core_env end def core_name_from_solr_url_env - Solr::Support::UrlHelper.core_name_from_url(ENV['SOLR_URL']) + if master_slave_configuration.master_slave_enabled? + Solr::Support::UrlHelper.core_name_from_url(master_slave_configuration.master_url) + elsif cloud_configuration.cloud_enabled? + raise 'Cloud is enabled, but no cloud configuration is set' + else + Solr::Support::UrlHelper.core_name_from_url(ENV['SOLR_URL']) + end end def build_env_url_core_config(name: nil) diff --git a/lib/solr/core_configuration/core_config.rb b/lib/solr/core_configuration/core_config.rb index f7f6aae..33440bb 100644 --- a/lib/solr/core_configuration/core_config.rb +++ b/lib/solr/core_configuration/core_config.rb @@ -18,7 +18,10 @@ def default? end def url - @url ||= File.join(Solr.configuration.url || ENV['SOLR_URL'], name.to_s).chomp('/') + @url ||= begin + url = Solr.configuration.url || ENV['SOLR_URL'] || ENV['SOLR_MASTER_URL'] + File.join(url, name.to_s).chomp('/') + end end def uri @@ -36,12 +39,14 @@ def initialize(name: nil, fields: {}) end def url - raise ArgumentError, "SOLR_URL can't be nil" if ENV['SOLR_URL'].nil? + raise ArgumentError, "SOLR_URL can't be nil" if ENV['SOLR_URL'].nil? && ENV['SOLR_MASTER_URL'].nil? + + url = ENV['SOLR_URL'] || ENV['SOLR_MASTER_URL'] if ENV['SOLR_CORE'] && ENV['SOLR_CORE'] != '' - File.join(*[ENV['SOLR_URL'], name.to_s].compact) + File.join(*[url, name.to_s].compact) else - ENV['SOLR_URL'] + url end end end diff --git a/lib/solr/support/url_helper.rb b/lib/solr/support/url_helper.rb index 5e841c0..6cb295b 100644 --- a/lib/solr/support/url_helper.rb +++ b/lib/solr/support/url_helper.rb @@ -54,7 +54,7 @@ def admin_base_url_for_cores if Solr.cloud_enabled? raise Solr::Errors::SolrQueryError, 'Core management is not supported in SolrCloud mode. Use collections API instead.' elsif Solr.master_slave_enabled? - Solr.configuration.master_url + solr_endpoint_from_url(Solr.configuration.master_url) else solr_endpoint_from_url(Solr.configuration.url || ENV['SOLR_URL']) end diff --git a/spec/solr/configuration_spec.rb b/spec/solr/configuration_spec.rb index 241bbf2..18ed4d9 100644 --- a/spec/solr/configuration_spec.rb +++ b/spec/solr/configuration_spec.rb @@ -72,7 +72,7 @@ Solr.configure do |config| config.faraday_configure do |f| f.adapter :net_http do |http| - http.idle_timeout = 150 + http.read_timeout = 150 end end end diff --git a/spec/solr/solrb_spec.rb b/spec/solr/solrb_spec.rb index 257fc73..8b9f215 100644 --- a/spec/solr/solrb_spec.rb +++ b/spec/solr/solrb_spec.rb @@ -4,9 +4,13 @@ end describe '.current_core_config' do - it 'uses default url' do + it 'uses default url', installation: :single do expect(Solr.current_core_config.url).to eq(File.join(*[ENV['SOLR_URL'], ENV['SOLR_CORE']].compact)) end + + it 'uses default url', installation: :master_replica do + expect(Solr.current_core_config.url).to eq(File.join(*[ENV['SOLR_MASTER_URL'], ENV['SOLR_CORE']].compact)) + end end describe '.with_core' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 90deba7..ee0f68e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -23,7 +23,25 @@ c.syntax = :expect end - config.after(:each) do - Solr.configuration = Solr::Configuration.new + config.before(:each) do + # if ENV['SOLR_MASTER_URL'] && ENV['SOLR_SLAVE_URL'] + # Solr.conS + # Solr.configuration = Solr::Configuration.new + # Solr.enable_master_slave! + # end + Solr.configure do |config| + if ENV['SOLR_MASTER_URL'] && ENV['SOLR_SLAVE_URL'] + config.master_url = ENV['SOLR_MASTER_URL'] + config.slave_url = ENV['SOLR_SLAVE_URL'] + end + + if ENV['SOLR_URL'] + config.url = ENV['SOLR_URL'] + end + end end + + # config.after(:each) do + # Solr.configuration = Solr::Configuration.new + # end end