NH-89406: improve control for building debug version of the extension and added documentation

.gitignore

@@ -48,6 +48,7 @@ test/clib/Makefile
 *.so
 *.o
 ext/oboe_metal/extconf_local.rb
+Makefile
 
 # test script
 install_gem.sh

.rubocop.yml

@@ -7,6 +7,7 @@ AllCops:
   NewCops: enable
   Exclude:
     - 'sample_start.rb'
+    - 'ext/oboe_metal/extconf_local.rb'
 
 Layout/LineLength:
   Enabled: false

Rakefile

@@ -33,11 +33,14 @@ Rake::TestTask.new do |t|
                    FileList['test/solarwinds_apm/*_test.rb'] +
                    FileList['test/opentelemetry/*_test.rb'] +
                    FileList['test/noop/*_test.rb'] +
+                   FileList['test/ext/*_test.rb'] +
                    FileList['test/support/*_test.rb'] -
                    FileList['test/support/swomarginalia/*_test.rb']
   end
 end
 
+################ Docker Task ################
+
 desc "Run an official docker ruby image with the specified tag. The test suite is launched if
 runtests is set to true, else a shell session is started for interactive test runs. The platform
 argument can be used to override the image architecture if multi-platform is supported.
@@ -71,29 +74,43 @@ desc 'Start ubuntu docker container for testing and debugging.'
 task docker_dev: [:docker_down] do
   cmd = "docker compose run --service-ports \
   --name ruby_sw_apm_ubuntu_development ruby_sw_apm_ubuntu_development"
-  Dir.chdir('test') do
-    sh cmd do |ok, res|
-      puts "ok: #{ok}, #{res.inspect}"
-    end
-  end
+  docker_cmd_execute(cmd)
+end
+
+desc 'Continue the docker container created last time'
+task :docker_con do
+  cmd = "docker container start ruby_sw_apm_ubuntu_development &&
+          docker exec -it ruby_sw_apm_ubuntu_development /bin/bash"
+  docker_cmd_execute(cmd)
+end
+
+desc 'Build the ubuntu docker container without cache'
+task :docker_build do
+  cmd = 'docker compose build --no-cache'
+  docker_cmd_execute(cmd)
 end
 
 desc 'Stop all containers that were started for testing and debugging'
 task :docker_down do
+  cmd = 'docker compose down -v --remove-orphans'
+  docker_cmd_execute(cmd)
+end
+
+def docker_cmd_execute(cmd)
   Dir.chdir('test') do
-    sh 'docker compose down -v --remove-orphans'
+    sh cmd do |ok, res|
+      puts "ok: #{ok}, #{res.inspect}"
+    end
   end
 end
 
+################ Build Gem Task ################
+
 desc 'alias for fetch_oboe_file_from_staging'
 task :fetch do
   Rake::Task['fetch_oboe_file'].invoke('stg')
 end
 
-@files = %w[oboe.h oboe_api.h oboe_api.cpp oboe.i oboe_debug.h bson/bson.h bson/platform_hacks.h]
-@ext_dir = File.expand_path('ext/oboe_metal')
-@ext_verify_dir = File.expand_path('ext/oboe_metal/verify')
-
 desc 'fetch oboe file from different environment'
 task :fetch_oboe_file, [:env] do |_t, args|
   abort('Missing env argument (abort)') if args['env'].nil? || args['env'].empty?
@@ -143,15 +160,7 @@ task :fetch_oboe_file, [:env] do |_t, args|
   files = %w[bson/bson.h bson/platform_hacks.h
              oboe.h oboe_api.h oboe_api.cpp oboe_debug.h oboe.i]
 
-  files.each do |filename|
-    remote_file = File.join(oboe_dir, 'include', filename)
-    local_file = File.join(ext_src_dir, filename)
-
-    puts "fetching #{remote_file}"
-    puts "      to #{local_file}"
-
-    IO.copy_stream(URI.parse(remote_file).open, local_file)
-  end
+  fetch_file_from_cloud(files, oboe_dir, ext_src_dir, 'include')
 
   sha_files = ['liboboe-1.0-lambda-x86_64.so.sha256',
                'liboboe-1.0-lambda-aarch64.so.sha256',
@@ -160,9 +169,20 @@ task :fetch_oboe_file, [:env] do |_t, args|
                'liboboe-1.0-alpine-x86_64.so.sha256',
                'liboboe-1.0-alpine-aarch64.so.sha256']
 
-  sha_files.each do |filename|
-    remote_file = File.join(oboe_dir, filename)
-    local_file  = File.join(ext_lib_dir, filename)
+  fetch_file_from_cloud(sha_files, oboe_dir, ext_lib_dir)
+
+  FileUtils.cd(ext_src_dir) do
+    sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i'
+    FileUtils.rm('oboe.i') if args['env'] != 'prod'
+  end
+
+  puts 'Fetching finished.'
+end
+
+def fetch_file_from_cloud(files, oboe_dir, dest_dir, folder = '')
+  files.each do |filename|
+    remote_file = File.join(oboe_dir, folder, filename)
+    local_file  = File.join(dest_dir, filename)
 
     puts "fetching #{remote_file}"
     puts "      to #{local_file}"
@@ -173,13 +193,6 @@ task :fetch_oboe_file, [:env] do |_t, args|
       puts "File #{remote_file} missing. #{e.message}"
     end
   end
-
-  FileUtils.cd(ext_src_dir) do
-    sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i'
-    FileUtils.rm('oboe.i') if args['env'] != 'prod'
-  end
-
-  puts 'Fetching finished.'
 end
 
 desc 'Build and publish to Rubygems'

ext/oboe_metal/README.md

@@ -17,10 +17,10 @@ rbenv which ruby    # => /home/wsh/.rbenv/versions/2.6.3/bin/ruby
 
 ## add debug info when compiling solarwinds_apm
 
-add this line to extconf.rb to turn off optimization
+enable `OBOE_DEBUG` to `true` to turn off optimization and enable debug information
 
-```ruby
-CONFIG["optflags"] = "-O0"
+```sh
+export OBOE_DEBUG=true
 ```
 
 ## start ruby app with gdb
@@ -65,3 +65,62 @@ Some inspiring examples here:
 <https://jvns.ca/blog/2016/06/12/a-weird-system-call-process-vm-readv/>
 
 <https://medium.com/@zanker/finding-a-ruby-bug-with-gdb-56d6b321bc86>
+
+<!-- markdownlint-disable MD025 -->
+
+# Debug the c-code with core dump
+
+<!-- markdownlint-enable MD025 -->
+
+If the core dump is produced, it's easier to check the backtrace with it in gdb
+
+For example, `(core dumped)` indicate the crash file is dumped
+
+```console
+./start.sh: line 37:    65 Segmentation fault      (core dumped) ...
+```
+
+## Prepare
+
+### 1. Check the core dump size is not constrained
+
+```console
+ulimit -c unlimited       # have to set this for unlimited core dump file size without trimmed error message
+```
+
+### 2. Check if the crash report program is configured correctly
+
+Ubuntu use [apport](https://wiki.ubuntu.com/Apport); Debian use [kdump](https://mudongliang.github.io/2018/07/02/debian-enable-kernel-dump.html)
+
+In ubuntu, if apport is disabled via `service apport stop`, the core dump file will be stored in the current directory and named `core`. If apport is enabled, find the crash file (typically under `/var/crash`) and extract the CoreDump file from it using `apport-unpack <filename>.crash <destination>`.
+
+### 3. Install solarwinds_apm with comprehensive debug information
+
+Set the environment variable `OBOE_DEBUG` to `true` that download the special version of liboboe.
+
+This liboboe is compiled with RelWithDebInfo flag on, which include the debug symbol and other debug information.
+
+```console
+export OBOE_DEBUG=true
+gem install solarwinds_apm
+```
+
+Reproduce the crash using this version of solarwinds_apm which provides extended debug information in the coredump.
+
+## Debug by checking the backtrace after obtain core dump file
+
+### 1. Check that ruby is debuggable
+
+Ensure that `ruby.h` is present by verifying the existence of the Ruby development library (e.g., `ruby-dev`).
+
+```console
+type ruby           # => ruby is hashed (/root/.rbenv/shims/ruby)
+rbenv which ruby    # => /root/.rbenv/versions/3.1.0/bin/ruby
+```
+
+### 2. Load the core dump file in gdb
+
+```console
+gdb /root/.rbenv/versions/3.1.0/bin/ruby core
+(gdb) bt full      # backtrace full trace; investigate the issue from here
+```

ext/oboe_metal/extconf.rb

@@ -18,22 +18,25 @@
 init_mkmf(CONFIG)
 
 ext_dir = __dir__
+oboe_debug = ENV['OBOE_DEBUG'].to_s.casecmp('true').zero?
+non_production = ENV['OBOE_DEV'].to_s.casecmp('true').zero? || ENV['OBOE_STAGING'].to_s.casecmp('true').zero?
 
-# Set the mkmf lib paths so we have no issues linking to
-# the SolarWindsAPM libs.
 swo_lib_dir = File.join(ext_dir, 'lib')
-swo_include = File.join(ext_dir, 'src')
+version     = File.read(File.join(ext_dir, 'src', 'VERSION')).strip
 
-# Download the appropriate liboboe from Staging or Production
-version = File.read(File.join(swo_include, 'VERSION')).strip
-if ENV['OBOE_DEV'].to_s.casecmp('true').zero?
+# OBOE_DEBUG has the highest priorities over oboe environment
+if oboe_debug
+  swo_path = File.join('https://agent-binaries.cloud.solarwinds.com/apm/c-lib/', version, 'relwithdebinfo')
+  puts 'Fetching c-lib from PRODUCTION DEBUG Build'
+elsif ENV['OBOE_DEV'].to_s.casecmp('true').zero?
   swo_path = 'https://solarwinds-apm-staging.s3.us-west-2.amazonaws.com/apm/c-lib/nightly'
   puts 'Fetching c-lib from DEVELOPMENT Build'
 elsif ENV['OBOE_STAGING'].to_s.casecmp('true').zero?
   swo_path = File.join('https://agent-binaries.global.st-ssp.solarwinds.com/apm/c-lib/', version)
-  puts 'Fetching c-lib from STAGING'
+  puts 'Fetching c-lib from STAGING Build'
 else
   swo_path = File.join('https://agent-binaries.cloud.solarwinds.com/apm/c-lib/', version)
+  puts 'Fetching c-lib from PRODUCTION Build'
 end
 
 swo_arch = 'x86_64'
@@ -70,23 +73,27 @@
   begin
     IO.copy_stream(URI.parse(swo_item).open, clib)
     clib_checksum = Digest::SHA256.file(clib).hexdigest
-    checksum      =  File.read(swo_checksum_file).strip
+    checksum      = File.read(swo_checksum_file).strip
+
+    # sha256 always from prod, so no matching for stg or nightly build or debug mode
+    # so ignore the sha comparsion when fetching from development and staging build
+    checksum = clib_checksum if non_production || oboe_debug
 
     # unfortunately these messages only show if the install command is run
     # with the `--verbose` flag
     if clib_checksum == checksum
       success = true
-      retries = 0
     else
+      puts 'Checksum Verification Fail' # this is mainly for testing
       warn '== ERROR ================================================================='
       warn 'Checksum Verification failed for the c-extension of the solarwinds_apm gem'
       warn 'Installation cannot continue'
       warn "\nChecksum packaged with gem:   #{checksum}"
       warn "Checksum calculated from lib: #{clib_checksum}"
       warn 'Contact [email protected] if the problem persists'
       warn '=========================================================================='
-      exit 1
     end
+    retries = 0
   rescue StandardError => e
     File.write(clib, '')
     retries -= 1
@@ -118,22 +125,26 @@
     $libs = append_library($libs, 'stdc++')
 
     $CFLAGS << " #{ENV.fetch('CFLAGS', nil)}"
-    # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11"
-    # TODO for debugging: -pg -gdwarf-2, remove for production
-    # -pg does not work on alpine https://www.openwall.com/lists/musl/2014/11/05/2
-    $CPPFLAGS << " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11  -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/include -I$$ORIGIN/../ext/oboe_metal/src"
-    # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/include"
+
+    # -pg option is used for generating profiling information with gprof
+    $CPPFLAGS << if oboe_debug
+                   " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11  -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/src"
+                 else
+                   " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/src"
+                 end
+
     $LIBS << " #{ENV.fetch('LIBS', nil)}"
 
-    # use "z,defs" to see what happens during linking
-    # $LDFLAGS << " #{ENV['LDFLAGS']} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib,-z,defs'  -lrt"
+    # -lrt option is used when linking programs with the GNU Compiler Collection (GCC) to
+    # include the POSIX real-time extensions library, librt.
     $LDFLAGS << " #{ENV.fetch('LDFLAGS', nil)} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib' -lrt"
     $CXXFLAGS += ' -std=c++11 '
 
-    # ____ include debug info, comment out when not debugging
-    # ____ -pg -> profiling info for gprof
-    CONFIG['debugflags'] = '-ggdb3 '
-    CONFIG['optflags'] = '-O0'
+    # OBOE_DEBUG need to be enabled before downloading and installing the gem
+    if oboe_debug
+      CONFIG['debugflags'] = '-ggdb3 '
+      CONFIG['optflags'] = '-O0'
+    end
 
     create_makefile('libsolarwinds_apm', 'src')
   else

solarwinds_apm.gemspec

@@ -33,13 +33,8 @@ Gem::Specification.new do |s|
               'ext/oboe_metal/src/bson/bson.h',
               'ext/oboe_metal/src/bson/platform_hacks.h',
               'ext/oboe_metal/src/init_solarwinds_apm.cc',
-              'ext/oboe_metal/src/VERSION',
-              'ext/oboe_metal/lib/liboboe-1.0-alpine-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-aarch64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-alpine-aarch64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-lambda-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-lambda-aarch64.so.sha256']
+              'ext/oboe_metal/src/VERSION']
+  s.files += Dir['ext/oboe_metal/lib/*']
 
   s.files -= ['Rakefile']
 

test/Dockerfile

@@ -77,7 +77,7 @@ RUN curl -SL https://github.com/swig/swig/archive/refs/tags/v4.0.2.tar.gz \
 
 # set up github package credentials for pushing
 RUN mkdir ~/.gem \
-    && echo -e "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \
+    && echo "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \
     && chmod 0600 ~/.gem/credentials
 
 ENV PATH="/usr/local/bin:/root/.rbenv/bin:/root/.rbenv/shims:$PATH"

test/docker-compose.yml

@@ -1,8 +1,6 @@
 # Copyright (c) 2023 SolarWinds, LLC.
 # All rights reserved.
 
-version: "2.1"
-
 #########################################################################################################
 #
 # docker-compose to set up the development container