diff --git a/.gitignore b/.gitignore index edd886b..434b72e 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ test/clib/Makefile *.so *.o ext/oboe_metal/extconf_local.rb +Makefile # test script install_gem.sh diff --git a/.rubocop.yml b/.rubocop.yml index bf12bb0..543cbdc 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,6 +7,7 @@ AllCops: NewCops: enable Exclude: - 'sample_start.rb' + - 'ext/oboe_metal/extconf_local.rb' Layout/LineLength: Enabled: false diff --git a/Rakefile b/Rakefile index 51e4bee..c183912 100755 --- a/Rakefile +++ b/Rakefile @@ -33,11 +33,14 @@ Rake::TestTask.new do |t| FileList['test/solarwinds_apm/*_test.rb'] + FileList['test/opentelemetry/*_test.rb'] + FileList['test/noop/*_test.rb'] + + FileList['test/ext/*_test.rb'] + FileList['test/support/*_test.rb'] - FileList['test/support/swomarginalia/*_test.rb'] end end +################ Docker Task ################ + desc "Run an official docker ruby image with the specified tag. The test suite is launched if runtests is set to true, else a shell session is started for interactive test runs. The platform argument can be used to override the image architecture if multi-platform is supported. @@ -71,29 +74,43 @@ desc 'Start ubuntu docker container for testing and debugging.' task docker_dev: [:docker_down] do cmd = "docker compose run --service-ports \ --name ruby_sw_apm_ubuntu_development ruby_sw_apm_ubuntu_development" - Dir.chdir('test') do - sh cmd do |ok, res| - puts "ok: #{ok}, #{res.inspect}" - end - end + docker_cmd_execute(cmd) +end + +desc 'Continue the docker container created last time' +task :docker_con do + cmd = "docker container start ruby_sw_apm_ubuntu_development && + docker exec -it ruby_sw_apm_ubuntu_development /bin/bash" + docker_cmd_execute(cmd) +end + +desc 'Build the ubuntu docker container without cache' +task :docker_build do + cmd = 'docker compose build --no-cache' + docker_cmd_execute(cmd) end desc 'Stop all containers that were started for testing and debugging' task :docker_down do + cmd = 'docker compose down -v --remove-orphans' + docker_cmd_execute(cmd) +end + +def docker_cmd_execute(cmd) Dir.chdir('test') do - sh 'docker compose down -v --remove-orphans' + sh cmd do |ok, res| + puts "ok: #{ok}, #{res.inspect}" + end end end +################ Build Gem Task ################ + desc 'alias for fetch_oboe_file_from_staging' task :fetch do Rake::Task['fetch_oboe_file'].invoke('stg') end -@files = %w[oboe.h oboe_api.h oboe_api.cpp oboe.i oboe_debug.h bson/bson.h bson/platform_hacks.h] -@ext_dir = File.expand_path('ext/oboe_metal') -@ext_verify_dir = File.expand_path('ext/oboe_metal/verify') - desc 'fetch oboe file from different environment' task :fetch_oboe_file, [:env] do |_t, args| abort('Missing env argument (abort)') if args['env'].nil? || args['env'].empty? @@ -143,15 +160,7 @@ task :fetch_oboe_file, [:env] do |_t, args| files = %w[bson/bson.h bson/platform_hacks.h oboe.h oboe_api.h oboe_api.cpp oboe_debug.h oboe.i] - files.each do |filename| - remote_file = File.join(oboe_dir, 'include', filename) - local_file = File.join(ext_src_dir, filename) - - puts "fetching #{remote_file}" - puts " to #{local_file}" - - IO.copy_stream(URI.parse(remote_file).open, local_file) - end + fetch_file_from_cloud(files, oboe_dir, ext_src_dir, 'include') sha_files = ['liboboe-1.0-lambda-x86_64.so.sha256', 'liboboe-1.0-lambda-aarch64.so.sha256', @@ -160,9 +169,20 @@ task :fetch_oboe_file, [:env] do |_t, args| 'liboboe-1.0-alpine-x86_64.so.sha256', 'liboboe-1.0-alpine-aarch64.so.sha256'] - sha_files.each do |filename| - remote_file = File.join(oboe_dir, filename) - local_file = File.join(ext_lib_dir, filename) + fetch_file_from_cloud(sha_files, oboe_dir, ext_lib_dir) + + FileUtils.cd(ext_src_dir) do + sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i' + FileUtils.rm('oboe.i') if args['env'] != 'prod' + end + + puts 'Fetching finished.' +end + +def fetch_file_from_cloud(files, oboe_dir, dest_dir, folder = '') + files.each do |filename| + remote_file = File.join(oboe_dir, folder, filename) + local_file = File.join(dest_dir, filename) puts "fetching #{remote_file}" puts " to #{local_file}" @@ -173,13 +193,6 @@ task :fetch_oboe_file, [:env] do |_t, args| puts "File #{remote_file} missing. #{e.message}" end end - - FileUtils.cd(ext_src_dir) do - sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i' - FileUtils.rm('oboe.i') if args['env'] != 'prod' - end - - puts 'Fetching finished.' end desc 'Build and publish to Rubygems' diff --git a/ext/oboe_metal/README.md b/ext/oboe_metal/README.md index b082574..35b14ae 100644 --- a/ext/oboe_metal/README.md +++ b/ext/oboe_metal/README.md @@ -17,10 +17,10 @@ rbenv which ruby # => /home/wsh/.rbenv/versions/2.6.3/bin/ruby ## add debug info when compiling solarwinds_apm -add this line to extconf.rb to turn off optimization +enable `OBOE_DEBUG` to `true` to turn off optimization and enable debug information -```ruby -CONFIG["optflags"] = "-O0" +```sh +export OBOE_DEBUG=true ``` ## start ruby app with gdb @@ -65,3 +65,62 @@ Some inspiring examples here: + + + +# Debug the c-code with core dump + + + +If the core dump is produced, it's easier to check the backtrace with it in gdb + +For example, `(core dumped)` indicate the crash file is dumped + +```console +./start.sh: line 37: 65 Segmentation fault (core dumped) ... +``` + +## Prepare + +### 1. Check the core dump size is not constrained + +```console +ulimit -c unlimited # have to set this for unlimited core dump file size without trimmed error message +``` + +### 2. Check if the crash report program is configured correctly + +Ubuntu use [apport](https://wiki.ubuntu.com/Apport); Debian use [kdump](https://mudongliang.github.io/2018/07/02/debian-enable-kernel-dump.html) + +In ubuntu, if apport is disabled via `service apport stop`, the core dump file will be stored in the current directory and named `core`. If apport is enabled, find the crash file (typically under `/var/crash`) and extract the CoreDump file from it using `apport-unpack .crash `. + +### 3. Install solarwinds_apm with comprehensive debug information + +Set the environment variable `OBOE_DEBUG` to `true` that download the special version of liboboe. + +This liboboe is compiled with RelWithDebInfo flag on, which include the debug symbol and other debug information. + +```console +export OBOE_DEBUG=true +gem install solarwinds_apm +``` + +Reproduce the crash using this version of solarwinds_apm which provides extended debug information in the coredump. + +## Debug by checking the backtrace after obtain core dump file + +### 1. Check that ruby is debuggable + +Ensure that `ruby.h` is present by verifying the existence of the Ruby development library (e.g., `ruby-dev`). + +```console +type ruby # => ruby is hashed (/root/.rbenv/shims/ruby) +rbenv which ruby # => /root/.rbenv/versions/3.1.0/bin/ruby +``` + +### 2. Load the core dump file in gdb + +```console +gdb /root/.rbenv/versions/3.1.0/bin/ruby core +(gdb) bt full # backtrace full trace; investigate the issue from here +``` diff --git a/ext/oboe_metal/extconf.rb b/ext/oboe_metal/extconf.rb index 6f252dc..3bd9f58 100644 --- a/ext/oboe_metal/extconf.rb +++ b/ext/oboe_metal/extconf.rb @@ -18,22 +18,25 @@ init_mkmf(CONFIG) ext_dir = __dir__ +oboe_debug = ENV['OBOE_DEBUG'].to_s.casecmp('true').zero? +non_production = ENV['OBOE_DEV'].to_s.casecmp('true').zero? || ENV['OBOE_STAGING'].to_s.casecmp('true').zero? -# Set the mkmf lib paths so we have no issues linking to -# the SolarWindsAPM libs. swo_lib_dir = File.join(ext_dir, 'lib') -swo_include = File.join(ext_dir, 'src') +version = File.read(File.join(ext_dir, 'src', 'VERSION')).strip -# Download the appropriate liboboe from Staging or Production -version = File.read(File.join(swo_include, 'VERSION')).strip -if ENV['OBOE_DEV'].to_s.casecmp('true').zero? +# OBOE_DEBUG has the highest priorities over oboe environment +if oboe_debug + swo_path = File.join('https://agent-binaries.cloud.solarwinds.com/apm/c-lib/', version, 'relwithdebinfo') + puts 'Fetching c-lib from PRODUCTION DEBUG Build' +elsif ENV['OBOE_DEV'].to_s.casecmp('true').zero? swo_path = 'https://solarwinds-apm-staging.s3.us-west-2.amazonaws.com/apm/c-lib/nightly' puts 'Fetching c-lib from DEVELOPMENT Build' elsif ENV['OBOE_STAGING'].to_s.casecmp('true').zero? swo_path = File.join('https://agent-binaries.global.st-ssp.solarwinds.com/apm/c-lib/', version) - puts 'Fetching c-lib from STAGING' + puts 'Fetching c-lib from STAGING Build' else swo_path = File.join('https://agent-binaries.cloud.solarwinds.com/apm/c-lib/', version) + puts 'Fetching c-lib from PRODUCTION Build' end swo_arch = 'x86_64' @@ -70,14 +73,18 @@ begin IO.copy_stream(URI.parse(swo_item).open, clib) clib_checksum = Digest::SHA256.file(clib).hexdigest - checksum = File.read(swo_checksum_file).strip + checksum = File.read(swo_checksum_file).strip + + # sha256 always from prod, so no matching for stg or nightly build or debug mode + # so ignore the sha comparsion when fetching from development and staging build + checksum = clib_checksum if non_production || oboe_debug # unfortunately these messages only show if the install command is run # with the `--verbose` flag if clib_checksum == checksum success = true - retries = 0 else + puts 'Checksum Verification Fail' # this is mainly for testing warn '== ERROR =================================================================' warn 'Checksum Verification failed for the c-extension of the solarwinds_apm gem' warn 'Installation cannot continue' @@ -85,8 +92,8 @@ warn "Checksum calculated from lib: #{clib_checksum}" warn 'Contact technicalsupport@solarwinds.com if the problem persists' warn '==========================================================================' - exit 1 end + retries = 0 rescue StandardError => e File.write(clib, '') retries -= 1 @@ -118,22 +125,26 @@ $libs = append_library($libs, 'stdc++') $CFLAGS << " #{ENV.fetch('CFLAGS', nil)}" - # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11" - # TODO for debugging: -pg -gdwarf-2, remove for production - # -pg does not work on alpine https://www.openwall.com/lists/musl/2014/11/05/2 - $CPPFLAGS << " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11 -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/include -I$$ORIGIN/../ext/oboe_metal/src" - # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/include" + + # -pg option is used for generating profiling information with gprof + $CPPFLAGS << if oboe_debug + " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11 -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/src" + else + " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/src" + end + $LIBS << " #{ENV.fetch('LIBS', nil)}" - # use "z,defs" to see what happens during linking - # $LDFLAGS << " #{ENV['LDFLAGS']} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib,-z,defs' -lrt" + # -lrt option is used when linking programs with the GNU Compiler Collection (GCC) to + # include the POSIX real-time extensions library, librt. $LDFLAGS << " #{ENV.fetch('LDFLAGS', nil)} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib' -lrt" $CXXFLAGS += ' -std=c++11 ' - # ____ include debug info, comment out when not debugging - # ____ -pg -> profiling info for gprof - CONFIG['debugflags'] = '-ggdb3 ' - CONFIG['optflags'] = '-O0' + # OBOE_DEBUG need to be enabled before downloading and installing the gem + if oboe_debug + CONFIG['debugflags'] = '-ggdb3 ' + CONFIG['optflags'] = '-O0' + end create_makefile('libsolarwinds_apm', 'src') else diff --git a/solarwinds_apm.gemspec b/solarwinds_apm.gemspec index fe4ce1b..ab79443 100644 --- a/solarwinds_apm.gemspec +++ b/solarwinds_apm.gemspec @@ -33,13 +33,8 @@ Gem::Specification.new do |s| 'ext/oboe_metal/src/bson/bson.h', 'ext/oboe_metal/src/bson/platform_hacks.h', 'ext/oboe_metal/src/init_solarwinds_apm.cc', - 'ext/oboe_metal/src/VERSION', - 'ext/oboe_metal/lib/liboboe-1.0-alpine-x86_64.so.sha256', - 'ext/oboe_metal/lib/liboboe-1.0-x86_64.so.sha256', - 'ext/oboe_metal/lib/liboboe-1.0-aarch64.so.sha256', - 'ext/oboe_metal/lib/liboboe-1.0-alpine-aarch64.so.sha256', - 'ext/oboe_metal/lib/liboboe-1.0-lambda-x86_64.so.sha256', - 'ext/oboe_metal/lib/liboboe-1.0-lambda-aarch64.so.sha256'] + 'ext/oboe_metal/src/VERSION'] + s.files += Dir['ext/oboe_metal/lib/*'] s.files -= ['Rakefile'] diff --git a/test/Dockerfile b/test/Dockerfile index 4061984..b1be0ee 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -77,7 +77,7 @@ RUN curl -SL https://github.com/swig/swig/archive/refs/tags/v4.0.2.tar.gz \ # set up github package credentials for pushing RUN mkdir ~/.gem \ - && echo -e "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \ + && echo "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \ && chmod 0600 ~/.gem/credentials ENV PATH="/usr/local/bin:/root/.rbenv/bin:/root/.rbenv/shims:$PATH" diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 3125f9c..f1f1101 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,8 +1,6 @@ # Copyright (c) 2023 SolarWinds, LLC. # All rights reserved. -version: "2.1" - ######################################################################################################### # # docker-compose to set up the development container diff --git a/test/ext/extconf_test.rb b/test/ext/extconf_test.rb new file mode 100644 index 0000000..3d4c3b3 --- /dev/null +++ b/test/ext/extconf_test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 SolarWinds, LLC. +# All rights reserved. + +require 'mkmf' +require 'minitest_helper' +require 'minitest/mock' +require 'uri' +require 'open-uri' + +describe 'extconf test' do + after do + ENV.delete('OBOE_DEBUG') + ENV.delete('OBOE_STAGING') + ENV.delete('OBOE_DEV') + end + + it 'simple_extconf_test_with_OBOE_DEBUG' do + ENV['OBOE_DEBUG'] = 'true' + output = stub_for_mkmf_test + assert_includes output, 'Fetching c-lib from PRODUCTION DEBUG Build' + end + + it 'OBOE_DEBUG_surpass_other_env' do + ENV['OBOE_DEBUG'] = 'true' + ENV['OBOE_STAGING'] = 'true' + output = stub_for_mkmf_test + assert_includes output, 'Fetching c-lib from PRODUCTION DEBUG Build' + end + + it 'simple_extconf_test_with_OBOE_STAGING' do + ENV['OBOE_STAGING'] = 'true' + output = stub_for_mkmf_test + assert_includes output, 'Fetching c-lib from STAGING Build' + end + + it 'simple_extconf_test_with_OBOE_DEV' do + ENV['OBOE_DEV'] = 'true' + output = stub_for_mkmf_test + assert_includes output, 'Fetching c-lib from DEVELOPMENT Build' + end + + it 'simple_extconf_test_with_no_env_variable' do + ENV.delete('OBOE_STAGING') + output = stub_for_mkmf_test + assert_includes output, 'Fetching c-lib from PRODUCTION Build' + assert_includes output, 'Checksum Verification Fail' + end +end diff --git a/test/minitest_helper.rb b/test/minitest_helper.rb index 1163d5e..21c70bc 100644 --- a/test/minitest_helper.rb +++ b/test/minitest_helper.rb @@ -359,3 +359,42 @@ def clean_old_setting ENV.delete('OTEL_TRACES_EXPORTER') ENV.delete('SW_APM_ENABLED') end + +# For mkmf test case +def capture_stdout_with_pipe + original_stdout = $stdout + read_io, write_io = IO.pipe + + # Redirect stderr to write_io (the writing end of the pipe) + $stdout = write_io + begin + yield # Run the code block that generates stderr output + ensure + $stdout = original_stdout # Restore the original stderr + write_io.close # Close the writing end of the pipe to stop sending data + end + + captured_output = read_io.read + read_io.close + captured_output +end + +def stub_for_mkmf_test + IO.stub(:copy_stream, nil) do + URI.stub(:parse, URI.parse('https://github.com')) do + URI::HTTPS.stub(:open, nil) do + File.stub(:symlink, nil) do + MakeMakefile.stub(:have_library, true) do + MakeMakefile.stub(:create_makefile, true) do + File.stub(:read, '1.2.3') do + capture_stdout_with_pipe do + load 'ext/oboe_metal/extconf.rb' + end + end + end + end + end + end + end + end +end diff --git a/test/test_setup.sh b/test/test_setup.sh index 5af3eb8..01b537e 100755 --- a/test/test_setup.sh +++ b/test/test_setup.sh @@ -14,7 +14,7 @@ if [ -r /etc/alpine-release ]; then echo "Tests do not work on aarch64 alpine, skipping." exit else - apk update && apk add --upgrade git ruby-dev g++ make curl bash perl zlib-dev linux-headers shared-mime-info sqlite-dev grpc + apk update && apk add --upgrade git ruby-dev g++ make curl bash perl zlib-dev linux-headers shared-mime-info sqlite-dev yaml-dev grpc fi elif [ -r /etc/debian_version ]; then # this is for ubuntu (> 22.04) and debian