From ab102ded607b64df3ba71eb0c84a8a7d248f2ace Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Mon, 3 Feb 2025 15:41:19 -0700 Subject: [PATCH 01/12] Fixed generate_api workflow The workflow missed `sudo apt-get install libcurl4-openssl-dev` to install `curb` on ubuntu. Signed-off-by: Theo Truong --- .github/workflows/generate_api.yml | 1 + .github/workflows/main.yml | 3 +-- .github/workflows/test-unreleased.yml | 2 +- Gemfile | 37 +++++++++++++++------------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate_api.yml b/.github/workflows/generate_api.yml index 5adece204..edcd6f48d 100644 --- a/.github/workflows/generate_api.yml +++ b/.github/workflows/generate_api.yml @@ -29,6 +29,7 @@ jobs: - name: Update bundler run: | sudo apt-get update + sudo apt-get install libcurl4-openssl-dev bundle install - name: Generate API diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5888ae8f4..364afafcb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,7 @@ jobs: env: TEST_OPENSEARCH_SERVER: http://localhost:9250 PORT: 9250 - FARADAY_VERSION: '~> 1.0' + FARADAY_VERSION: '~> 1' strategy: fail-fast: false matrix: @@ -106,7 +106,6 @@ jobs: run: | sudo apt-get update sudo apt-get install libcurl4-openssl-dev - sudo apt-get install ca-certificates ruby -v bundle install - name: Test Client Security diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 036bc2e49..476bdcc17 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -24,7 +24,7 @@ jobs: entry: - { ruby_version: '3.3', opensearch_ref: '1.x', jdk_version: '11' } - { ruby_version: '3.3', opensearch_ref: '2.x', jdk_version: '17' } - - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '17' } + - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '23' } steps: - uses: ruby/setup-ruby@v1 diff --git a/Gemfile b/Gemfile index 1edfa3201..7061cf664 100644 --- a/Gemfile +++ b/Gemfile @@ -28,19 +28,16 @@ source 'https://rubygems.org' gem 'opensearch-ruby', path: __dir__, require: false +# TODO: remove unnecessary dependencies gem 'ansi' gem 'bundler' gem 'cane' -gem 'faraday-httpclient' -gem 'faraday-net_http_persistent' gem 'hashie' -gem 'httpclient' gem 'jbuilder' gem 'jsonify' gem 'minitest', '~> 5' gem 'minitest-reporters', '~> 1' gem 'mocha', '~> 2' -gem 'net-http-persistent' gem 'pry' gem 'rake', '~> 13' gem 'rspec', '~> 3' @@ -50,19 +47,27 @@ gem 'rubocop-rspec' gem 'shoulda-context' gem 'simplecov', '~> 0.17', '< 0.18' gem 'test-unit', '~> 2' -gem 'typhoeus', '~> 1.4' gem 'webmock', '~> 2.0' gem 'yard' -gem 'curb' unless defined? JRUBY_VERSION -gem 'faraday-patron' unless defined? JRUBY_VERSION -gem 'patron' unless defined? JRUBY_VERSION +if defined? JRUBY_VERSION + gem 'manticore' + gem 'pry-nav' +else + gem 'curb' + gem 'require-prof' + gem 'ruby-prof' +end -gem 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius) -gem 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius) - -gem 'manticore' if defined? JRUBY_VERSION -gem 'pry-nav' if defined? JRUBY_VERSION - -gem 'faraday', ENV.fetch('FARADAY_VERSION', nil), require: false if ENV.key?('FARADAY_VERSION') -gem 'faraday-typhoeus' if !ENV.key?('FARADAY_VERSION') && Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') +if ENV.key?('FARADAY_VERSION') + gem 'faraday', ENV.fetch('FARADAY_VERSION'), require: false + gem 'httpclient' + gem 'net-http-persistent' + gem 'patron' unless defined? JRUBY_VERSION + gem 'typhoeus' +else + gem 'faraday-httpclient' + gem 'faraday-net_http_persistent' + gem 'faraday-patron' unless defined? JRUBY_VERSION + gem 'faraday-typhoeus' +end From e4d52fd97f19606d403860d8ccc70031fac2858f Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 09:08:30 -0700 Subject: [PATCH 02/12] # Signed-off-by: Theo Truong --- .github/workflows/test-unreleased.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 476bdcc17..b1e5e2c23 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -42,28 +42,28 @@ jobs: working-directory: opensearch run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT - - name: Restore cached build - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: opensearch/distribution/archives/linux-tar/build/distributions - key: ${{ steps.get-key.outputs.key }} +# - name: Restore cached build +# id: cache-restore +# uses: actions/cache/restore@v4 +# with: +# path: opensearch/distribution/archives/linux-tar/build/distributions +# key: ${{ steps.get-key.outputs.key }} - name: Setup Java JDK uses: actions/setup-java@v4 - if: steps.cache-restore.outputs.cache-hit != 'true' +# if: steps.cache-restore.outputs.cache-hit != 'true' with: distribution: 'temurin' java-version: ${{ matrix.entry.jdk_version }} - name: Assemble OpenSearch - if: steps.cache-restore.outputs.cache-hit != 'true' +# if: steps.cache-restore.outputs.cache-hit != 'true' working-directory: opensearch run: ./gradlew :distribution:archives:linux-tar:assemble --stacktrace - name: Save cached build - if: steps.cache-restore.outputs.cache-hit != 'true' +# if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: opensearch/distribution/archives/linux-tar/build/distributions From c7c0af536fb5bfdb2c10f5d89582c140e0b18925 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 11:33:39 -0700 Subject: [PATCH 03/12] # Signed-off-by: Theo Truong --- .github/workflows/test-unreleased.yml | 2 +- spec/opensearch/transport/client_spec.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index b1e5e2c23..8f94ba0e1 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -24,7 +24,7 @@ jobs: entry: - { ruby_version: '3.3', opensearch_ref: '1.x', jdk_version: '11' } - { ruby_version: '3.3', opensearch_ref: '2.x', jdk_version: '17' } - - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '23' } + - { ruby_version: '3.3', opensearch_ref: 'main', jdk_version: '21' } steps: - uses: ruby/setup-ruby@v1 diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index b380b1164..60039b25d 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1809,7 +1809,9 @@ it 'keeps connections open' do response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] + puts 'RELOADING CONNECTIONS' client.transport.reload_connections! + puts 'CONNECTIONS RELOADED' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) From b91db36cd4e034a8e7adad66525159546efc0892 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 11:43:09 -0700 Subject: [PATCH 04/12] # Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index 60039b25d..d5a0a0a35 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1802,16 +1802,16 @@ client.transport.connections.first.connection.builder.adapter end - it 'uses the patron connection handler' do + it 'uses the typhoeus connection handler' do expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] - puts 'RELOADING CONNECTIONS' + puts 'RELOADING CONNECTIONS', Time.now client.transport.reload_connections! - puts 'CONNECTIONS RELOADED' + puts 'CONNECTIONS RELOADED', Time.now response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) From c01987c4005cf7fe6a91307846b5852961576c2a Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 11:51:25 -0700 Subject: [PATCH 05/12] # Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index d5a0a0a35..afb301b6e 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1803,15 +1803,15 @@ end it 'uses the typhoeus connection handler' do + puts 'TESTING TYPHOEUS', Time.now + puts adapter expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] - puts 'RELOADING CONNECTIONS', Time.now client.transport.reload_connections! - puts 'CONNECTIONS RELOADED', Time.now response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) From 96f79fc9fc7864d7a80023521573c4b7a21d2be7 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:03:11 -0700 Subject: [PATCH 06/12] # Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index afb301b6e..a1d72c333 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1803,15 +1803,16 @@ end it 'uses the typhoeus connection handler' do - puts 'TESTING TYPHOEUS', Time.now - puts adapter expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do + puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] + puts 'Start reloading connections' client.transport.reload_connections! + puts 'Connections reloaded' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) From 07995eec6f8cbfd4bab3960fab277b1221f50173 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:28:55 -0700 Subject: [PATCH 07/12] # Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 27 ++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index a1d72c333..fe0c79e61 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1762,57 +1762,58 @@ end end - context 'when patron is used as an adapter', unless: jruby? do + + context 'when typhoeus is used as an adapter', unless: jruby? do before do - require 'patron' + require 'typhoeus' end let(:options) do - { adapter: :patron } + { adapter: :typhoeus } end let(:adapter) do client.transport.connections.first.connection.builder.adapter end - it 'uses the patron connection handler' do - expect(adapter).to eq('Faraday::Adapter::Patron') + it 'uses the typhoeus connection handler' do + expect(adapter).to eq('Faraday::Adapter::Typhoeus') end it 'keeps connections open' do + puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] + puts 'Start reloading connections' client.transport.reload_connections! + puts 'Connections reloaded' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) end end - context 'when typhoeus is used as an adapter', unless: jruby? do + context 'when patron is used as an adapter', unless: jruby? do before do - require 'typhoeus' + require 'patron' end let(:options) do - { adapter: :typhoeus } + { adapter: :patron } end let(:adapter) do client.transport.connections.first.connection.builder.adapter end - it 'uses the typhoeus connection handler' do - expect(adapter).to eq('Faraday::Adapter::Typhoeus') + it 'uses the patron connection handler' do + expect(adapter).to eq('Faraday::Adapter::Patron') end it 'keeps connections open' do - puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] - puts 'Start reloading connections' client.transport.reload_connections! - puts 'Connections reloaded' response = client.perform_request('GET', '_nodes/stats/http') connections_after = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] expect(connections_after).to be >= (connections_before) From 00765d702b1f76848212f4dd0a0b0ef11eb6c343 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:33:07 -0700 Subject: [PATCH 08/12] # Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index fe0c79e61..61dd2340f 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1762,7 +1762,6 @@ end end - context 'when typhoeus is used as an adapter', unless: jruby? do before do require 'typhoeus' From 06eb0c03e45a362b3d5df44c2be5a2091f2e0c18 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:43:21 -0700 Subject: [PATCH 09/12] # Signed-off-by: Theo Truong --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 7061cf664..7d8464312 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ if ENV.key?('FARADAY_VERSION') gem 'httpclient' gem 'net-http-persistent' gem 'patron' unless defined? JRUBY_VERSION - gem 'typhoeus' + gem 'typhoeus', '~> 1.4' else gem 'faraday-httpclient' gem 'faraday-net_http_persistent' From 1945396fac7631227c9a39b6e9accc01826344a3 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:52:05 -0700 Subject: [PATCH 10/12] # Testing with _cat/health Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index 61dd2340f..dcf74bae4 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1780,6 +1780,8 @@ end it 'keeps connections open' do + puts 'Testing with _cat/health' + puts client.cat.health puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] From 2b4ac90f7116b73538c9801ee8237aacc72030e1 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 12:57:00 -0700 Subject: [PATCH 11/12] # Testing with _cat/health Signed-off-by: Theo Truong --- spec/opensearch/transport/client_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/opensearch/transport/client_spec.rb b/spec/opensearch/transport/client_spec.rb index dcf74bae4..6c5a8b3bc 100644 --- a/spec/opensearch/transport/client_spec.rb +++ b/spec/opensearch/transport/client_spec.rb @@ -1781,7 +1781,7 @@ it 'keeps connections open' do puts 'Testing with _cat/health' - puts client.cat.health + puts client.perform_request('GET', '_cat/health') puts 'Performing GET _nodes/stats/http' response = client.perform_request('GET', '_nodes/stats/http') connections_before = response.body['nodes'].values.find { |n| n['name'] == node_names.first }['http']['total_opened'] From cb8dfff2b35ec06d157d1477694c6807a576ec27 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 5 Feb 2025 14:50:03 -0700 Subject: [PATCH 12/12] # Restore cache Signed-off-by: Theo Truong --- .github/workflows/test-unreleased.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-unreleased.yml b/.github/workflows/test-unreleased.yml index 8f94ba0e1..f43c6417b 100644 --- a/.github/workflows/test-unreleased.yml +++ b/.github/workflows/test-unreleased.yml @@ -42,28 +42,28 @@ jobs: working-directory: opensearch run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT -# - name: Restore cached build -# id: cache-restore -# uses: actions/cache/restore@v4 -# with: -# path: opensearch/distribution/archives/linux-tar/build/distributions -# key: ${{ steps.get-key.outputs.key }} + - name: Restore cached build + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} - name: Setup Java JDK uses: actions/setup-java@v4 -# if: steps.cache-restore.outputs.cache-hit != 'true' + if: steps.cache-restore.outputs.cache-hit != 'true' with: distribution: 'temurin' java-version: ${{ matrix.entry.jdk_version }} - name: Assemble OpenSearch -# if: steps.cache-restore.outputs.cache-hit != 'true' + if: steps.cache-restore.outputs.cache-hit != 'true' working-directory: opensearch run: ./gradlew :distribution:archives:linux-tar:assemble --stacktrace - name: Save cached build -# if: steps.cache-restore.outputs.cache-hit != 'true' + if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: opensearch/distribution/archives/linux-tar/build/distributions