NH-89406: improve control for building debug version of the extension and added documentation

.gitignore

@@ -47,6 +47,8 @@ appoptics.com
 test/clib/Makefile
 *.so
 *.o
+*.so.debug
+ext/oboe_metal/extconf_local.rb
 
 # test script
 install_gem.sh

.rubocop.yml

@@ -7,6 +7,7 @@ AllCops:
   NewCops: enable
   Exclude:
     - 'sample_start.rb'
+    - 'ext/oboe_metal/extconf_local.rb'
 
 Layout/LineLength:
   Enabled: false

Rakefile

@@ -78,6 +78,16 @@ task docker_dev: [:docker_down] do
   end
 end
 
+desc 'Start ubuntu docker container for testing and debugging.'
+task :docker_build do
+  cmd = 'docker compose build --no-cache'
+  Dir.chdir('test') do
+    sh cmd do |ok, res|
+      puts "ok: #{ok}, #{res.inspect}"
+    end
+  end
+end
+
 desc 'Stop all containers that were started for testing and debugging'
 task :docker_down do
   Dir.chdir('test') do
@@ -90,10 +100,6 @@ task :fetch do
   Rake::Task['fetch_oboe_file'].invoke('stg')
 end
 
-@files = %w[oboe.h oboe_api.h oboe_api.cpp oboe.i oboe_debug.h bson/bson.h bson/platform_hacks.h]
-@ext_dir = File.expand_path('ext/oboe_metal')
-@ext_verify_dir = File.expand_path('ext/oboe_metal/verify')
-
 desc 'fetch oboe file from different environment'
 task :fetch_oboe_file, [:env] do |_t, args|
   abort('Missing env argument (abort)') if args['env'].nil? || args['env'].empty?
@@ -143,15 +149,7 @@ task :fetch_oboe_file, [:env] do |_t, args|
   files = %w[bson/bson.h bson/platform_hacks.h
              oboe.h oboe_api.h oboe_api.cpp oboe_debug.h oboe.i]
 
-  files.each do |filename|
-    remote_file = File.join(oboe_dir, 'include', filename)
-    local_file = File.join(ext_src_dir, filename)
-
-    puts "fetching #{remote_file}"
-    puts "      to #{local_file}"
-
-    IO.copy_stream(URI.parse(remote_file).open, local_file)
-  end
+  fetch_file_from_cloud(files, oboe_dir, ext_src_dir, 'include')
 
   sha_files = ['liboboe-1.0-lambda-x86_64.so.sha256',
                'liboboe-1.0-lambda-aarch64.so.sha256',
@@ -160,9 +158,20 @@ task :fetch_oboe_file, [:env] do |_t, args|
                'liboboe-1.0-alpine-x86_64.so.sha256',
                'liboboe-1.0-alpine-aarch64.so.sha256']
 
-  sha_files.each do |filename|
-    remote_file = File.join(oboe_dir, filename)
-    local_file  = File.join(ext_lib_dir, filename)
+  fetch_file_from_cloud(sha_files, oboe_dir, ext_lib_dir)
+
+  FileUtils.cd(ext_src_dir) do
+    sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i'
+    FileUtils.rm('oboe.i') if args['env'] != 'prod'
+  end
+
+  puts 'Fetching finished.'
+end
+
+def fetch_file_from_cloud(files, oboe_dir, dest_dir, folder = '')
+  files.each do |filename|
+    remote_file = File.join(oboe_dir, folder, filename)
+    local_file  = File.join(dest_dir, filename)
 
     puts "fetching #{remote_file}"
     puts "      to #{local_file}"
@@ -173,13 +182,6 @@ task :fetch_oboe_file, [:env] do |_t, args|
       puts "File #{remote_file} missing. #{e.message}"
     end
   end
-
-  FileUtils.cd(ext_src_dir) do
-    sh 'swig -c++ -ruby -module oboe_metal -o oboe_swig_wrap.cc oboe.i'
-    FileUtils.rm('oboe.i') if args['env'] != 'prod'
-  end
-
-  puts 'Fetching finished.'
 end
 
 desc 'Build and publish to Rubygems'

ext/oboe_metal/README.md

@@ -65,3 +65,86 @@ Some inspiring examples here:
 <https://jvns.ca/blog/2016/06/12/a-weird-system-call-process-vm-readv/>
 
 <https://medium.com/@zanker/finding-a-ruby-bug-with-gdb-56d6b321bc86>
+
+## Debug the c-code with core dump
+
+If the core dump is produced, it's easier to check the backtrace with it in gdb
+
+For example, `(core dumped)` indicate the crash file is dumped
+
+```console
+./start.sh: line 37:    65 Segmentation fault      (core dumped) ...
+```
+
+### Prepare
+
+#### 1. Install solarwinds_apm with debug symbol on
+
+```console
+export OBOE_DEBUG=true  # enable debug flag when compiling; and also download *.debug into lib/ when create the gem for ease of use
+export OBOE_DEV=true    # optional: if you want to install the nightly build liboboe
+
+gem install solarwinds_apm
+```
+
+#### 2. Check the core dump size is not constrained
+
+```console
+ulimit -c unlimited       # have to set this for unlimited core dump file size without trimmed error message
+```
+
+#### 3. Check if the crash report program is configured correctly
+
+Ubuntu use [apport](https://wiki.ubuntu.com/Apport); Debian use [kdump](https://mudongliang.github.io/2018/07/02/debian-enable-kernel-dump.html)
+
+In ubuntu, if disable the apport by `service apport stop`, the core dump file will be stored in the current directory and named as `core`
+
+#### 4. Gather the `*.debug` file for oboe symbol
+
+Assume the gem has following file structure
+
+```console
+root@docker:~/.rbenv/versions/3.1.0/lib/ruby/gems/3.1.0/gems/solarwinds_apm-6.0.6# tree .
+|-- ext
+|   `-- oboe_metal
+|       |-- init_solarwinds_apm.o
+|       |-- lib
+|       |   |-- liboboe-1.0-aarch64.so
+|       |   |-- liboboe-1.0.so.0 -> liboboe-1.0-aarch64.so
+|       |   `-- liboboe.so -> liboboe-1.0-aarch64.so
+|       |-- libsolarwinds_apm.so
+|       |-- oboe_api.o
+|       |-- oboe_swig_wrap.o
+|       `-- src
+|           |-- ...
+`-- lib
+    |-- libsolarwinds_apm.so
+    |-- oboe_metal.rb
+    |-- rails
+    |   ...
+    |-- solarwinds_apm
+    |   ...
+    `-- solarwinds_apm.rb
+
+18 directories, 79 files
+```
+
+The `*.debug` file need to be stored in ext/oboe_metal/lib/folder, then start the gdb
+
+The `*.debug` file has to match the exact build of liboboe (e.g. version, system, etc.) to avoid CRC mismatch
+
+### Debug by checking the backtrace after obtain core dump file
+
+#### 1. Check that ruby is debuggable
+
+```console
+type ruby           # => ruby is hashed (/root/.rbenv/shims/ruby)
+rbenv which ruby    # => /root/.rbenv/versions/3.1.0/bin/ruby
+```
+
+#### 2. Load the core dump file in gdb
+
+```console
+gdb /root/.rbenv/versions/3.1.0/bin/ruby core
+(gdb) bt full      # backtrace full trace; investigate the issue from here
+```

ext/oboe_metal/extconf.rb

@@ -22,18 +22,16 @@
 # Set the mkmf lib paths so we have no issues linking to
 # the SolarWindsAPM libs.
 swo_lib_dir = File.join(ext_dir, 'lib')
-swo_include = File.join(ext_dir, 'src')
-
-# Download the appropriate liboboe from Staging or Production
-version = File.read(File.join(swo_include, 'VERSION')).strip
+version     = File.read(File.join(ext_dir, 'src', 'VERSION')).strip
 if ENV['OBOE_DEV'].to_s.casecmp('true').zero?
   swo_path = 'https://solarwinds-apm-staging.s3.us-west-2.amazonaws.com/apm/c-lib/nightly'
   puts 'Fetching c-lib from DEVELOPMENT Build'
 elsif ENV['OBOE_STAGING'].to_s.casecmp('true').zero?
   swo_path = File.join('https://agent-binaries.global.st-ssp.solarwinds.com/apm/c-lib/', version)
-  puts 'Fetching c-lib from STAGING'
+  puts 'Fetching c-lib from STAGING Build'
 else
   swo_path = File.join('https://agent-binaries.cloud.solarwinds.com/apm/c-lib/', version)
+  puts 'Fetching c-lib from PRODUCTION Build'
 end
 
 swo_arch = 'x86_64'
@@ -66,6 +64,14 @@
 
 retries = 3
 success = false
+
+# sha256 always from prod, so no matching for stg or nightly build
+# so ignore the sha comparsion when fetching from development and staging build
+if ENV['OBOE_DEV'].to_s.casecmp('true').zero? || ENV['OBOE_STAGING'].to_s.casecmp('true').zero?
+  success = true
+  retries = 0
+end
+
 while retries.positive?
   begin
     IO.copy_stream(URI.parse(swo_item).open, clib)
@@ -118,22 +124,27 @@
     $libs = append_library($libs, 'stdc++')
 
     $CFLAGS << " #{ENV.fetch('CFLAGS', nil)}"
-    # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11"
-    # TODO for debugging: -pg -gdwarf-2, remove for production
-    # -pg does not work on alpine https://www.openwall.com/lists/musl/2014/11/05/2
-    $CPPFLAGS << " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11  -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/include -I$$ORIGIN/../ext/oboe_metal/src"
-    # $CPPFLAGS << " #{ENV['CPPFLAGS']} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/include"
+
+    # -pg option is used for generating profiling information with gprof
+    $CPPFLAGS << if ENV['OBOE_DEBUG'].to_s.casecmp('true').zero?
+                   " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11  -gdwarf-2 -I$$ORIGIN/../ext/oboe_metal/src"
+                 else
+                   " #{ENV.fetch('CPPFLAGS', nil)} -std=c++11 -I$$ORIGIN/../ext/oboe_metal/src"
+                 end
+
     $LIBS << " #{ENV.fetch('LIBS', nil)}"
 
-    # use "z,defs" to see what happens during linking
-    # $LDFLAGS << " #{ENV['LDFLAGS']} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib,-z,defs'  -lrt"
+    # -lrt option is used when linking programs with the GNU Compiler Collection (GCC) to
+    # include the POSIX real-time extensions library, librt.
     $LDFLAGS << " #{ENV.fetch('LDFLAGS', nil)} '-Wl,-rpath=$$ORIGIN/../ext/oboe_metal/lib' -lrt"
     $CXXFLAGS += ' -std=c++11 '
 
     # ____ include debug info, comment out when not debugging
-    # ____ -pg -> profiling info for gprof
-    CONFIG['debugflags'] = '-ggdb3 '
-    CONFIG['optflags'] = '-O0'
+    # OBOE_DEBUG need to be enabled before downloading and installing the gem
+    if ENV['OBOE_DEBUG'].to_s.casecmp('true').zero?
+      CONFIG['debugflags'] = '-ggdb3 '
+      CONFIG['optflags'] = '-O0'
+    end
 
     create_makefile('libsolarwinds_apm', 'src')
   else

solarwinds_apm.gemspec

@@ -33,13 +33,8 @@ Gem::Specification.new do |s|
               'ext/oboe_metal/src/bson/bson.h',
               'ext/oboe_metal/src/bson/platform_hacks.h',
               'ext/oboe_metal/src/init_solarwinds_apm.cc',
-              'ext/oboe_metal/src/VERSION',
-              'ext/oboe_metal/lib/liboboe-1.0-alpine-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-aarch64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-alpine-aarch64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-lambda-x86_64.so.sha256',
-              'ext/oboe_metal/lib/liboboe-1.0-lambda-aarch64.so.sha256']
+              'ext/oboe_metal/src/VERSION']
+  s.files += Dir['ext/oboe_metal/lib/*']
 
   s.files -= ['Rakefile']
 

test/Dockerfile

@@ -77,7 +77,7 @@ RUN curl -SL https://github.com/swig/swig/archive/refs/tags/v4.0.2.tar.gz \
 
 # set up github package credentials for pushing
 RUN mkdir ~/.gem \
-    && echo -e "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \
+    && echo "---\n:github: Bearer ${BUNDLE_RUBYGEMS__PKG__GITHUB__COM}" >> ~/.gem/credentials \
     && chmod 0600 ~/.gem/credentials
 
 ENV PATH="/usr/local/bin:/root/.rbenv/bin:/root/.rbenv/shims:$PATH"