From 7ab6760d62b71f7a0ad5036850b2eb6f7a314f98 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Wed, 20 Jul 2022 20:28:07 -0500 Subject: [PATCH 01/70] imp(regex): initial regex implementation --- Cargo.toml | 1 + examples/regex.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++++ src/pyroscope.rs | 29 ++++++++++++++++++++++ src/session.rs | 9 ++++++- 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 examples/regex.rs diff --git a/Cargo.toml b/Cargo.toml index 8f18a360..fb57d349 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ names = "0.13.0" reqwest = { version = "0.11", features = ["blocking", "rustls-tls-native-roots"]} url = "2.2.2" libc = "^0.2.124" +regex = "1" [dev-dependencies] tokio = { version = "1.18", features = ["full"] } diff --git a/examples/regex.rs b/examples/regex.rs new file mode 100644 index 00000000..f263413e --- /dev/null +++ b/examples/regex.rs @@ -0,0 +1,61 @@ +extern crate pyroscope; + +use pyroscope::{PyroscopeAgent, Result}; +use pyroscope_pprofrs::{pprof_backend, PprofConfig}; +use std::hash::{Hash, Hasher}; + +fn hash_rounds(n: u64) -> u64 { + let hash_str = "Some string to hash"; + let mut default_hasher = std::collections::hash_map::DefaultHasher::new(); + + for _ in 0..n { + for _ in 0..1000 { + default_hasher.write(hash_str.as_bytes()); + } + hash_str.hash(&mut default_hasher); + } + + n +} + +fn main() -> Result<()> { + let agent = PyroscopeAgent::builder("http://localhost:4040", "example.regex") + .backend(pprof_backend(PprofConfig::new().sample_rate(100))) + .tags([("TagA", "ValueA"), ("TagB", "ValueB")].to_vec()) + .regex(regex::Regex::new(r"std::").unwrap()) + .build()?; + + // Show start time + let start = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + println!("Start Time: {}", start); + + // Start Agent + let agent_running = agent.start()?; + + let _result = hash_rounds(300_000); + + // Show stop time + let stop = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + println!("Stop Time: {}", stop); + + // Stop Agent + let agent_ready = agent_running.stop()?; + + // Shutdown the Agent + agent_ready.shutdown(); + + // Show program exit time + let exit = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + println!("Exit Time: {}", exit); + + Ok(()) +} diff --git a/src/pyroscope.rs b/src/pyroscope.rs index 239fe0d6..e86609ec 100644 --- a/src/pyroscope.rs +++ b/src/pyroscope.rs @@ -1,3 +1,4 @@ +use regex::Regex; use std::{ collections::HashMap, marker::PhantomData, @@ -42,6 +43,8 @@ pub struct PyroscopeConfig { pub spy_name: String, /// Authentication Token pub auth_token: Option, + /// Regex to apply + pub regex: Option, } impl Default for PyroscopeConfig { @@ -56,6 +59,7 @@ impl Default for PyroscopeConfig { sample_rate: 100u32, spy_name: "undefined".to_string(), auth_token: None, + regex: None, } } } @@ -76,6 +80,7 @@ impl PyroscopeConfig { sample_rate: 100u32, // Default sample rate spy_name: String::from("undefined"), // Spy Name should be set by the backend auth_token: None, // No authentication token + regex: None, // No regex } } @@ -116,6 +121,14 @@ impl PyroscopeConfig { } } + /// Set the Regex. + pub fn regex(self, regex: Regex) -> Self { + Self { + regex: Some(regex), + ..self + } + } + /// Set the tags. /// /// # Example @@ -243,6 +256,22 @@ impl PyroscopeAgentBuilder { } } + /// Set Regex. + /// This is optional. If not set, the agent will not apply any regex. + /// #Example + /// ```ignore + /// let builder = PyroscopeAgentBuilder::new("http://localhost:8080", "my-app") + /// .regex(Regex::new("^my-app.*").unwrap()) + /// .build() + /// ?; + /// ``` + pub fn regex(self, regex: Regex) -> Self { + Self { + config: self.config.regex(regex), + ..self + } + } + /// Set tags. Default is empty. /// /// # Example diff --git a/src/session.rs b/src/session.rs index f1537418..79831139 100644 --- a/src/session.rs +++ b/src/session.rs @@ -151,7 +151,14 @@ impl Session { ); // Convert a report to a byte array - let report_u8 = report.to_string().into_bytes(); + let mut report_string = report.to_string(); + + // Apply Regex to the report + if let Some(regex) = self.config.regex.clone() { + report_string = regex.replace_all(&report_string, "").to_string(); + } + + let report_u8 = report_string.into_bytes(); // Check if the report is empty if report_u8.is_empty() { From 62b3c25a819dd9a482f10b6c1bcedbfc76c1ac0d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Wed, 20 Jul 2022 23:21:56 -0500 Subject: [PATCH 02/70] imp(ruby): add regex support --- pyroscope_ffi/ruby/ext/rbspy/Cargo.toml | 1 + pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 10 ++++++++-- pyroscope_ffi/ruby/lib/pyroscope.rb | 6 ++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml b/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml index 2f321dd3..6207cb65 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml +++ b/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml @@ -14,6 +14,7 @@ pyroscope = { path = "../../../../" } pyroscope_rbspy = { path = "../../../../pyroscope_backends/pyroscope_rbspy" } ffikit = { path = "../../../ffikit" } pretty_env_logger = "0.4.0" +regex = "1" [patch.crates-io] read-process-memory = {git = "https://github.com/omarabid/read-process-memory.git", branch = "0.1.4-fix"} diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index b4faabb2..b37b4fc7 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -10,8 +10,8 @@ use std::os::raw::c_char; #[no_mangle] pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, - sample_rate: u32, detect_subprocesses: bool, on_cpu: bool, report_pid: bool, - report_thread_id: bool, tags: *const c_char, + directory_name: *const c_char, sample_rate: u32, detect_subprocesses: bool, on_cpu: bool, + report_pid: bool, report_thread_id: bool, tags: *const c_char, ) -> bool { // Initialize FFIKit let recv = ffikit::initialize_ffi().unwrap(); @@ -31,6 +31,11 @@ pub extern "C" fn initialize_agent( .unwrap() .to_string(); + let directory_name = unsafe { CStr::from_ptr(directory_name) } + .to_str() + .unwrap() + .to_string(); + let tags_string = unsafe { CStr::from_ptr(tags) } .to_str() .unwrap() @@ -52,6 +57,7 @@ pub extern "C" fn initialize_agent( let mut agent_builder = PyroscopeAgent::builder(server_address, application_name) .backend(rbspy) + .regex(regex::Regex::new(&directory_name).unwrap()) .tags(tags); if auth_token != "" { diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index 11929703..e79a47e0 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -4,7 +4,7 @@ module Pyroscope module Rust extend FFI::Library ffi_lib File.expand_path(File.dirname(__FILE__)) + "/rbspy/rbspy.#{RbConfig::CONFIG["DLEXT"]}" - attach_function :initialize_agent, [:string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool + attach_function :initialize_agent, [:string, :string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool attach_function :add_thread_tag, [:uint64, :string, :string], :bool attach_function :remove_thread_tag, [:uint64, :string, :string], :bool attach_function :add_global_tag, [:string, :string], :bool @@ -18,11 +18,12 @@ module Utils attach_function :thread_id, [], :uint64 end - Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :log_level, :tags) do + Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :directory_name, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :log_level, :tags) do def initialize(*) self.application_name = '' self.server_address = 'http://localhost:4040' self.auth_token = '' + self.directory_name = __dir__ self.sample_rate = 100 self.detect_subprocesses = false self.on_cpu = true @@ -45,6 +46,7 @@ def configure @config.app_name || @config.application_name || "", @config.server_address || "", @config.auth_token || "", + @config.directory_name || __dir__, @config.sample_rate || 100, @config.detect_subprocesses || false, @config.on_cpu || false, From feaffcdb8991b414d1fd56d4c92ac5c9df5ba3fa Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 21 Jul 2022 10:10:08 -0500 Subject: [PATCH 03/70] imp(lib): add report transformation function --- src/pyroscope.rs | 32 +++++++++++++++++++++++++++++++- src/session.rs | 10 +++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/pyroscope.rs b/src/pyroscope.rs index e86609ec..dc083070 100644 --- a/src/pyroscope.rs +++ b/src/pyroscope.rs @@ -10,7 +10,7 @@ use std::{ }; use crate::{ - backend::{void_backend, BackendReady, BackendUninitialized, Rule, Tag, VoidConfig}, + backend::{void_backend, BackendReady, BackendUninitialized, Report, Rule, Tag, VoidConfig}, error::Result, session::{Session, SessionManager, SessionSignal}, timer::{Timer, TimerSignal}, @@ -45,6 +45,8 @@ pub struct PyroscopeConfig { pub auth_token: Option, /// Regex to apply pub regex: Option, + /// Function to apply + pub func: Option Report>, } impl Default for PyroscopeConfig { @@ -60,6 +62,7 @@ impl Default for PyroscopeConfig { spy_name: "undefined".to_string(), auth_token: None, regex: None, + func: None, } } } @@ -81,6 +84,7 @@ impl PyroscopeConfig { spy_name: String::from("undefined"), // Spy Name should be set by the backend auth_token: None, // No authentication token regex: None, // No regex + func: None, // No function } } @@ -129,6 +133,14 @@ impl PyroscopeConfig { } } + /// Set the Function. + pub fn func(self, func: fn(Report) -> Report) -> Self { + Self { + func: Some(func), + ..self + } + } + /// Set the tags. /// /// # Example @@ -272,6 +284,24 @@ impl PyroscopeAgentBuilder { } } + /// Set the Function. + /// This is optional. If not set, the agent will not apply any function. + /// #Example + /// ```ignore + /// let builder = PyroscopeAgentBuilder::new("http://localhost:8080", "my-app") + /// .func(|report| { + /// report + /// }) + /// .build() + /// ?; + /// ``` + pub fn func(self, func: fn(Report) -> Report) -> Self { + Self { + config: self.config.func(func), + ..self + } + } + /// Set tags. Default is empty. /// /// # Example diff --git a/src/session.rs b/src/session.rs index 79831139..c9022ba6 100644 --- a/src/session.rs +++ b/src/session.rs @@ -150,8 +150,16 @@ impl Session { self.until ); + // own the report + let mut report_owned = report.to_owned(); + + // Apply function to the report + if let Some(func) = self.config.func.clone() { + report_owned = func(report_owned); + } + // Convert a report to a byte array - let mut report_string = report.to_string(); + let mut report_string = report_owned.to_string(); // Apply Regex to the report if let Some(regex) = self.config.regex.clone() { From 557c0fd00f75a66e68175a4652ca0f7eeeaf023b Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 21 Jul 2022 10:56:54 -0500 Subject: [PATCH 04/70] imp(ruby): add report transformation function --- pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 54 ++++++++++++++++++++++++- pyroscope_ffi/ruby/lib/pyroscope.rb | 4 +- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index b37b4fc7..9fb6817b 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -1,5 +1,5 @@ use ffikit::Signal; -use pyroscope::backend::Tag; +use pyroscope::backend::{Report, StackFrame, Tag}; use pyroscope::PyroscopeAgent; use pyroscope_rbspy::{rbspy_backend, RbspyConfig}; use std::collections::hash_map::DefaultHasher; @@ -7,6 +7,48 @@ use std::ffi::CStr; use std::hash::Hasher; use std::os::raw::c_char; +pub fn transform_report(report: Report) -> Report { + let data = report + .data + .iter() + .map(|(stacktrace, count)| { + let new_frames = stacktrace + .frames + .iter() + .map(|frame| { + let frame = frame.to_owned(); + let regex = regex::Regex::new(r"(.+?/gems/|.+?/ruby/)").unwrap(); + let new_filename = Some( + regex + .replace_all(frame.filename.unwrap().as_str(), "") + .to_string(), + ); + + // something + StackFrame::new( + frame.module, + frame.name, + new_filename, + frame.relative_path, + frame.absolute_path, + frame.line, + ) + }) + .collect(); + + let mut mystack = stacktrace.to_owned(); + + mystack.frames = new_frames; + + (mystack, count.to_owned()) + }) + .collect(); + + let new_report = Report::new(data).metadata(report.metadata.clone()); + + new_report +} + #[no_mangle] pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, @@ -55,9 +97,17 @@ pub extern "C" fn initialize_agent( let tags = string_to_tags(tags_ref); let rbspy = rbspy_backend(rbspy_config); + let mut regex_pattern = String::from(r""); + + if directory_name != String::from(".") { + regex_pattern.push_str(directory_name.as_str()); + regex_pattern.push_str(r"/"); + } + let mut agent_builder = PyroscopeAgent::builder(server_address, application_name) .backend(rbspy) - .regex(regex::Regex::new(&directory_name).unwrap()) + .func(transform_report) + .regex(regex::Regex::new(regex_pattern.as_str()).unwrap()) .tags(tags); if auth_token != "" { diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index e79a47e0..8f8b0327 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -23,7 +23,7 @@ def initialize(*) self.application_name = '' self.server_address = 'http://localhost:4040' self.auth_token = '' - self.directory_name = __dir__ + self.directory_name = File.dirname($0) self.sample_rate = 100 self.detect_subprocesses = false self.on_cpu = true @@ -46,7 +46,7 @@ def configure @config.app_name || @config.application_name || "", @config.server_address || "", @config.auth_token || "", - @config.directory_name || __dir__, + @config.directory_name || File.dirname($0), @config.sample_rate || 100, @config.detect_subprocesses || false, @config.on_cpu || false, From 1f921cbc0f90ee23a8f843069867c7278227e5d7 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 21 Jul 2022 10:58:42 -0500 Subject: [PATCH 05/70] chore(release): ruby gem 0.3.2 --- pyroscope_ffi/ruby/lib/pyroscope/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyroscope_ffi/ruby/lib/pyroscope/version.rb b/pyroscope_ffi/ruby/lib/pyroscope/version.rb index 45cde9a1..445cf038 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope/version.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope/version.rb @@ -1,3 +1,3 @@ module Pyroscope - VERSION = '0.3.1'.freeze + VERSION = '0.3.2'.freeze end From ca93e1e1d9f1289d6711b0843b65bbc4834beb98 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 21 Jul 2022 20:33:16 -0700 Subject: [PATCH 06/70] ruby substitute function changes --- .github/workflows/ruby.yml | 4 +- Cargo.toml | 7 ++- pyroscope_ffi/ruby/ext/rbspy/Cargo.toml | 1 - pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 58 +++++++++++++++---------- pyroscope_ffi/ruby/lib/pyroscope.rb | 6 +-- src/pyroscope.rs | 29 ------------- src/session.rs | 8 +--- 7 files changed, 44 insertions(+), 69 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 764a6ce7..15898f53 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -128,9 +128,9 @@ jobs: fail-fast: false matrix: include: - - macos-version: "10.15" + - macos-version: "11.0" target: x86_64-apple-darwin - py-platform: macosx-10_15_x86_64 + py-platform: macosx-11_0_x86_64 - macos-version: "11.0" target: aarch64-apple-darwin py-platform: macosx-11_0_arm64 diff --git a/Cargo.toml b/Cargo.toml index fb57d349..780874f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,9 @@ documentation = "https://docs.rs/pyroscope" repository = "https://github.com/pyroscope-io/pyroscope-rs" readme = "README.md" autobins = false -autoexamples = true -autotests = true -autobenches = true +autoexamples = true +autotests = true +autobenches = true [workspace] members = [ @@ -57,7 +57,6 @@ names = "0.13.0" reqwest = { version = "0.11", features = ["blocking", "rustls-tls-native-roots"]} url = "2.2.2" libc = "^0.2.124" -regex = "1" [dev-dependencies] tokio = { version = "1.18", features = ["full"] } diff --git a/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml b/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml index 6207cb65..2f321dd3 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml +++ b/pyroscope_ffi/ruby/ext/rbspy/Cargo.toml @@ -14,7 +14,6 @@ pyroscope = { path = "../../../../" } pyroscope_rbspy = { path = "../../../../pyroscope_backends/pyroscope_rbspy" } ffikit = { path = "../../../ffikit" } pretty_env_logger = "0.4.0" -regex = "1" [patch.crates-io] read-process-memory = {git = "https://github.com/omarabid/read-process-memory.git", branch = "0.1.4-fix"} diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index 9fb6817b..6f045a66 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -6,8 +6,12 @@ use std::collections::hash_map::DefaultHasher; use std::ffi::CStr; use std::hash::Hasher; use std::os::raw::c_char; +use std::env; pub fn transform_report(report: Report) -> Report { + let cwd = env::current_dir().unwrap(); + let cwd = cwd.to_str().unwrap_or(""); + let data = report .data .iter() @@ -17,18 +21,41 @@ pub fn transform_report(report: Report) -> Report { .iter() .map(|frame| { let frame = frame.to_owned(); - let regex = regex::Regex::new(r"(.+?/gems/|.+?/ruby/)").unwrap(); - let new_filename = Some( - regex - .replace_all(frame.filename.unwrap().as_str(), "") - .to_string(), - ); + let mut s = frame.filename.unwrap(); + match s.find(cwd) { + Some(i) => { + s = s[(i+cwd.len()+1)..].to_string(); + } + None => { + match s.find("/gems/") { + Some(i) => { + s = s[(i+1)..].to_string(); + } + None => { + match s.find("/ruby/") { + Some(i) => { + s = s[(i+6)..].to_string(); + match s.find("/") { + Some(i) => { + s = s[(i+1)..].to_string(); + } + None => { + } + } + } + None => { + } + } + } + } + } + } // something StackFrame::new( frame.module, frame.name, - new_filename, + Some(s.to_string()), frame.relative_path, frame.absolute_path, frame.line, @@ -52,8 +79,8 @@ pub fn transform_report(report: Report) -> Report { #[no_mangle] pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, - directory_name: *const c_char, sample_rate: u32, detect_subprocesses: bool, on_cpu: bool, - report_pid: bool, report_thread_id: bool, tags: *const c_char, + sample_rate: u32, detect_subprocesses: bool, on_cpu: bool, report_pid: bool, + report_thread_id: bool, tags: *const c_char, ) -> bool { // Initialize FFIKit let recv = ffikit::initialize_ffi().unwrap(); @@ -73,11 +100,6 @@ pub extern "C" fn initialize_agent( .unwrap() .to_string(); - let directory_name = unsafe { CStr::from_ptr(directory_name) } - .to_str() - .unwrap() - .to_string(); - let tags_string = unsafe { CStr::from_ptr(tags) } .to_str() .unwrap() @@ -97,17 +119,9 @@ pub extern "C" fn initialize_agent( let tags = string_to_tags(tags_ref); let rbspy = rbspy_backend(rbspy_config); - let mut regex_pattern = String::from(r""); - - if directory_name != String::from(".") { - regex_pattern.push_str(directory_name.as_str()); - regex_pattern.push_str(r"/"); - } - let mut agent_builder = PyroscopeAgent::builder(server_address, application_name) .backend(rbspy) .func(transform_report) - .regex(regex::Regex::new(regex_pattern.as_str()).unwrap()) .tags(tags); if auth_token != "" { diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index 8f8b0327..11929703 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -4,7 +4,7 @@ module Pyroscope module Rust extend FFI::Library ffi_lib File.expand_path(File.dirname(__FILE__)) + "/rbspy/rbspy.#{RbConfig::CONFIG["DLEXT"]}" - attach_function :initialize_agent, [:string, :string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool + attach_function :initialize_agent, [:string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool attach_function :add_thread_tag, [:uint64, :string, :string], :bool attach_function :remove_thread_tag, [:uint64, :string, :string], :bool attach_function :add_global_tag, [:string, :string], :bool @@ -18,12 +18,11 @@ module Utils attach_function :thread_id, [], :uint64 end - Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :directory_name, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :log_level, :tags) do + Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :log_level, :tags) do def initialize(*) self.application_name = '' self.server_address = 'http://localhost:4040' self.auth_token = '' - self.directory_name = File.dirname($0) self.sample_rate = 100 self.detect_subprocesses = false self.on_cpu = true @@ -46,7 +45,6 @@ def configure @config.app_name || @config.application_name || "", @config.server_address || "", @config.auth_token || "", - @config.directory_name || File.dirname($0), @config.sample_rate || 100, @config.detect_subprocesses || false, @config.on_cpu || false, diff --git a/src/pyroscope.rs b/src/pyroscope.rs index dc083070..08127a42 100644 --- a/src/pyroscope.rs +++ b/src/pyroscope.rs @@ -1,4 +1,3 @@ -use regex::Regex; use std::{ collections::HashMap, marker::PhantomData, @@ -43,8 +42,6 @@ pub struct PyroscopeConfig { pub spy_name: String, /// Authentication Token pub auth_token: Option, - /// Regex to apply - pub regex: Option, /// Function to apply pub func: Option Report>, } @@ -61,7 +58,6 @@ impl Default for PyroscopeConfig { sample_rate: 100u32, spy_name: "undefined".to_string(), auth_token: None, - regex: None, func: None, } } @@ -83,7 +79,6 @@ impl PyroscopeConfig { sample_rate: 100u32, // Default sample rate spy_name: String::from("undefined"), // Spy Name should be set by the backend auth_token: None, // No authentication token - regex: None, // No regex func: None, // No function } } @@ -125,14 +120,6 @@ impl PyroscopeConfig { } } - /// Set the Regex. - pub fn regex(self, regex: Regex) -> Self { - Self { - regex: Some(regex), - ..self - } - } - /// Set the Function. pub fn func(self, func: fn(Report) -> Report) -> Self { Self { @@ -268,22 +255,6 @@ impl PyroscopeAgentBuilder { } } - /// Set Regex. - /// This is optional. If not set, the agent will not apply any regex. - /// #Example - /// ```ignore - /// let builder = PyroscopeAgentBuilder::new("http://localhost:8080", "my-app") - /// .regex(Regex::new("^my-app.*").unwrap()) - /// .build() - /// ?; - /// ``` - pub fn regex(self, regex: Regex) -> Self { - Self { - config: self.config.regex(regex), - ..self - } - } - /// Set the Function. /// This is optional. If not set, the agent will not apply any function. /// #Example diff --git a/src/session.rs b/src/session.rs index c9022ba6..cea971a2 100644 --- a/src/session.rs +++ b/src/session.rs @@ -159,13 +159,7 @@ impl Session { } // Convert a report to a byte array - let mut report_string = report_owned.to_string(); - - // Apply Regex to the report - if let Some(regex) = self.config.regex.clone() { - report_string = regex.replace_all(&report_string, "").to_string(); - } - + let report_string = report_owned.to_string(); let report_u8 = report_string.into_bytes(); // Check if the report is empty From 4a3c12055c258558582e330c3315d115ad292c27 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 22 Jul 2022 14:13:57 -0500 Subject: [PATCH 07/70] refactor(examples): change regex example --- examples/{regex.rs => transform.rs} | 41 ++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) rename examples/{regex.rs => transform.rs} (58%) diff --git a/examples/regex.rs b/examples/transform.rs similarity index 58% rename from examples/regex.rs rename to examples/transform.rs index f263413e..a5c3bf6e 100644 --- a/examples/regex.rs +++ b/examples/transform.rs @@ -1,6 +1,9 @@ extern crate pyroscope; -use pyroscope::{PyroscopeAgent, Result}; +use pyroscope::{ + backend::{Report, StackFrame}, + PyroscopeAgent, Result, +}; use pyroscope_pprofrs::{pprof_backend, PprofConfig}; use std::hash::{Hash, Hasher}; @@ -18,11 +21,43 @@ fn hash_rounds(n: u64) -> u64 { n } +pub fn transform_report(report: Report) -> Report { + let data = report + .iter() + .map(|(stacktrace, count)| { + let new_frames = stacktrace + .iter() + .map(|frame| { + let frame = frame.clone(); + // something + StackFrame::new( + frame.module, + frame.name, + frame.filename, + frame.relative_path, + frame.absolute_path, + frame.line, + ) + }) + .collect(); + + let mut mystack = stacktrace.to_owned(); + + mystack.frames = new_frames; + + (mystack, count.to_owned()) + }) + .collect(); + + let new_report = Report::new(data).metadata(report.metadata.clone()); + + new_report +} fn main() -> Result<()> { - let agent = PyroscopeAgent::builder("http://localhost:4040", "example.regex") + let agent = PyroscopeAgent::builder("http://localhost:4040", "example.transform") .backend(pprof_backend(PprofConfig::new().sample_rate(100))) .tags([("TagA", "ValueA"), ("TagB", "ValueB")].to_vec()) - .regex(regex::Regex::new(r"std::").unwrap()) + .func(transform_report) .build()?; // Show start time From 4d0c6c117b9f1ca66e29c1397ea1e3371d652d1d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 22 Jul 2022 14:14:16 -0500 Subject: [PATCH 08/70] imp(lib): add iter function to Report and StackTrace --- src/backend/types.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/backend/types.rs b/src/backend/types.rs index 1b0657f1..984ae666 100644 --- a/src/backend/types.rs +++ b/src/backend/types.rs @@ -147,6 +147,11 @@ impl Report { } } + /// Return an iterator over the StackTraces of the Report. + pub fn iter(&self) -> impl Iterator { + self.data.iter() + } + /// Set the metadata of the report. pub fn metadata(self, metadata: Metadata) -> Self { Self { @@ -254,6 +259,11 @@ impl StackTrace { metadata, } } + + /// Return an iterator over the frames of the stacktrace. + pub fn iter(&self) -> impl Iterator { + self.frames.iter() + } } /// StackFrame From 2d3b11690e1088ba82750c986926ca4dad6d6d3e Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 23 Jul 2022 22:10:01 -0500 Subject: [PATCH 09/70] refactor(session): minor refactoring --- src/session.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/session.rs b/src/session.rs index cea971a2..15e79e1d 100644 --- a/src/session.rs +++ b/src/session.rs @@ -86,7 +86,9 @@ impl SessionManager { /// Used to contain the session data, and send it to the server. #[derive(Clone, Debug)] pub struct Session { + // Pyroscope instance configuration pub config: PyroscopeConfig, + // Session data pub reports: Vec, // unix time pub from: u64, @@ -159,8 +161,7 @@ impl Session { } // Convert a report to a byte array - let report_string = report_owned.to_string(); - let report_u8 = report_string.into_bytes(); + let report_u8 = report_owned.to_string().into_bytes(); // Check if the report is empty if report_u8.is_empty() { @@ -180,8 +181,7 @@ impl Session { )?; // Parse URL - let parsed_url = Url::parse(&url)?; - let joined = parsed_url.join("ingest")?; + let joined = Url::parse(&url)?.join("ingest")?; // Create Reqwest builder let mut req_builder = client @@ -189,10 +189,6 @@ impl Session { .header("Content-Type", "binary/octet-stream"); // Set authentication token - //if self.config.auth_token.is_some() { - //req_builder = req_builder.bearer_auth(self.config.auth_token.clone().unwrap()); - //} - // rewrite with let some if let Some(auth_token) = self.config.auth_token.clone() { req_builder = req_builder.bearer_auth(auth_token); } From ee3a3dbf0c0f774b66954e7b545a5892c7a5ec90 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 23 Jul 2022 22:54:26 -0500 Subject: [PATCH 10/70] docs(README): update README files --- README.md | 9 +++- pyroscope_ffi/python/README.md | 61 ++++++++++++++++++-------- pyroscope_ffi/ruby/README.md | 80 ++++++++++++++++++++-------------- 3 files changed, 99 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 6ad6da1e..cc0e6af3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Add this to your `Cargo.toml`: ```toml [dependencies] -pyroscope = "0.5.3" +pyroscope = "0.5.4" pyroscope_pprofrs = "0.2" ``` @@ -79,6 +79,13 @@ The Pyroscope Agent doesn't do any profiling. The agent role is to orchasrate a - [rbspy](pyroscope_backends/pyroscope_rbspy): Ruby Profiler. A wrapper around [rbspy](https://rbspy.github.io/). - [py-spy](pyroscope_backends/pyroscope_pyspy): Python Profiler. A wrapper around [py-spy](https://github.com/benfred/py-spy). +### Native Integration + +Pyroscope can be used directly from your projects with native integration. No agent or external programs are required. + +- [Python](https://pypi.org/project/pyroscope-io/): Python Package. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/python#readme)|[Documentation](https://pyroscope.io/docs/python/) +- [Ruby](https://rubygems.org/gems/pyroscope): Ruby Gem. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/ruby#readme)|[Documentation](https://pyroscope.io/docs/ruby/) + ### Limitations diff --git a/pyroscope_ffi/python/README.md b/pyroscope_ffi/python/README.md index 9a4ba39f..b5246199 100644 --- a/pyroscope_ffi/python/README.md +++ b/pyroscope_ffi/python/README.md @@ -1,4 +1,15 @@ -# Pyroscope Python Integration +# Pyroscope Python Package + +**Pyroscope integration for Python** + +[![license](https://img.shields.io/badge/license-Apache2.0-blue.svg)](LICENSE) +![tests](https://github.com/pyroscope-io/pyroscope-rs/workflows/Tests/badge.svg) +![build](https://github.com/pyroscope-io/pyroscope-rs/workflows/Build/badge.svg) +[![PyPI version](https://badge.fury.io/py/pyroscope-io.svg)](https://badge.fury.io/py/pyroscope-io) +[![PyPI](https://img.shields.io/pypi/pyversions/pyroscope-io.svg?maxAge=2592000)](https://pypi.python.org/pypi/pyroscope-io) + + +--- ### What is Pyroscope [Pyroscope](https://github.com/pyroscope-io/pyroscope) is a tool that lets you continuously profile your applications to prevent and debug performance issues in your code. It consists of a low-overhead agent which sends data to the Pyroscope server which includes a custom-built storage engine. This allows for you to store and query any applications profiling data in an extremely efficient and cost effective way. @@ -9,37 +20,53 @@ pip install pyroscope-io ``` -### Basic Usage of Pyroscope -``` -import pyroscope_io as pyroscope +### Minimal Configuration + +Add the following code to your application. This code will initialize pyroscope profiler and start profiling: + +```python +import pyroscope pyroscope.configure( - application_name = "my.python.app", # replace this with some name for your application - server_address = "http://my-pyroscope-server:4040", # replace this with the address of your pyroscope server + application_name = "my.python.app", # replace this with some name for your application + server_address = "http://my-pyroscope-server:4040", # replace this with the address of your pyroscope server ) ``` -### Adding Tags -Tags allow for users to view their data at different levels of granularity depending on what "slices" make sense for their application. This can be anything from region or microservice to more dynamic tags like controller or api route. +### Full Configuration -``` -import os +Optionally, you can configure several parameters: + +```python import pyroscope pyroscope.configure( - application_name = "simple.python.app", - server_address = "http://my-pyroscope-server:4040", - - tags = { - "hostname": os.getenv("HOSTNAME"), + application_name = "my.python.app", # replace this with some name for your application + server_address = "http://my-pyroscope-server:4040", # replace this with the address of your pyroscope server + auth_token = "{YOUR_API_KEY}", # optional, if authentication is enabled, specify the API key + sample_rate = 100, # default is 100 + detect_subprocesses = False, # detect subprocesses started by the main process; default is False + oncpu = True # report cpu time only; default is True + native = False # profile native extensions; default is False + gil_only = True # only include traces for threads that are holding on to the Global Interpreter Lock; default is True + log_level = "info" # default is info, possible values: trace, debug, info, warn, error and critical + tags = { + "region": '{os.getenv("REGION")}', } ) +``` + +### Tags + +You can add tags to certain parts of your code: + +```python # You can use a wrapper: with pyroscope.tag_wrapper({ "controller": "slow_controller_i_want_to_profile" }): slow_code() ``` +### Example -### Examples -For more examples see [examples/python](https://github.com/pyroscope-io/pyroscope/tree/main/examples/python) in the main repo. +Check out this [example python project in our repository](https://github.com/pyroscope-io/pyroscope/tree/main/examples/python) for examples of how you can use these features. diff --git a/pyroscope_ffi/ruby/README.md b/pyroscope_ffi/ruby/README.md index 53f87732..12ad1de3 100644 --- a/pyroscope_ffi/ruby/README.md +++ b/pyroscope_ffi/ruby/README.md @@ -1,57 +1,71 @@ -Pyroscope Ruby Integration --Beta-- -===================================== +# Pyroscope Ruby Gem -**note**: This is a beta release. It requires local compilation, might be -buggy and is frequently updated. For the initial implementation, find it [here](https://github.com/pyroscope-io/pyroscope-ruby). Please report any [issues](https://github.com/pyroscope-io/pyroscope-rs/issues). +**Pyroscope integration for Ruby** -## Installation +[![license](https://img.shields.io/badge/license-Apache2.0-blue.svg)](LICENSE) +![tests](https://github.com/pyroscope-io/pyroscope-rs/workflows/Tests/badge.svg) +![build](https://github.com/pyroscope-io/pyroscope-rs/workflows/Build/badge.svg) +[![Gem version](https://badge.fury.io/rb/pyroscope.svg)](https://badge.fury.io/rb/pyroscope) -1. You need the Rust toolchain to compile the library locally. To install - Rust: +--- -``` -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y -export PATH=$PATH:/root/.cargo/bin -``` +### What is Pyroscope +[Pyroscope](https://github.com/pyroscope-io/pyroscope) is a tool that lets you continuously profile your applications to prevent and debug performance issues in your code. It consists of a low-overhead agent which sends data to the Pyroscope server which includes a custom-built storage engine. This allows for you to store and query any applications profiling data in an extremely efficient and cost effective way. -2. Building/Insalling from Rubygems -``` -gem install pyroscope_beta -``` +### Supported platforms -3. Building/Installing from source +| Linux | macOS | Windows | Docker | +|:-----:|:-----:|:-------:|:------:| +| ✅ | ✅ | | ✅ | -Change directory to `pyroscope_ffi/ruby` and run +### Profiling Ruby applications +Add the `pyroscope` gem to your Gemfile: + +```bash +bundle add pyroscope ``` -gem build pyroscope.gemspec -gem install ./pyroscope.gemspec -``` -## Configuration +### Basic Configuration + +Add the following code to your application. If you're using rails, put this into `config/initializers` directory. This code will initialize pyroscope profiler and start profiling: -Configuration is similar to the old package except for `application_name`: +```ruby +require 'pyroscope' +Pyroscope.configure do |config| + config.application_name = "my.ruby.app" # replace this with some name for your application + config.server_address = "http://my-pyroscope-server:4040" # replace this with the address of your pyroscope server + # config.auth_token = "{YOUR_API_KEY}" # optionally, if authentication is enabled, specify the API key +end ``` -require 'pyroscope_beta' + +### Tags + +Pyroscope ruby integration provides a number of ways to tag profiling data. For example, you can provide tags when you're initializing the profiler: + +```ruby +require 'pyroscope' Pyroscope.configure do |config| - config.application_name = "ruby.app" - config.server_address = "http://localhost:4040" - config.detect_subprocesses = true + config.application_name = "my.ruby.app" + config.server_address = "http://my-pyroscope-server:4040" + config.tags = { - :key => "value", + "hostname" => ENV["HOSTNAME"], } end ``` -## Adding tags +or you can dynamically tag certain parts of your code: -Tags passed to configure are global. To tag code locally, you can use: - -``` -Pyroscope.tag_wrapper({"profile": "profile-1"}) do - // Tagged profile +```ruby +Pyroscope.tag_wrapper({ "controller": "slow_controller_i_want_to_profile" }) do + slow_code end ``` + +### Example + +Check out this [example ruby project in our repository](https://github.com/pyroscope-io/pyroscope/tree/main/examples/ruby) for examples of how you can use these features. From ab65eb2d03e71d1396a50fd4ddcf911717eee0d3 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Wed, 27 Jul 2022 22:16:58 -0500 Subject: [PATCH 11/70] imp(python): add enable_logging option --- pyroscope_ffi/python/pyroscope/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pyroscope_ffi/python/pyroscope/__init__.py b/pyroscope_ffi/python/pyroscope/__init__.py index b378d16a..62fbe40d 100644 --- a/pyroscope_ffi/python/pyroscope/__init__.py +++ b/pyroscope_ffi/python/pyroscope/__init__.py @@ -6,10 +6,11 @@ from contextlib import contextmanager -Config = namedtuple('Config', ('app_name', 'application_name', 'server_address', 'auth_token', 'sample_rate', 'detect_subprocesses','oncpu', 'native', 'gil_only')) +Config = namedtuple('Config', ('app_name', 'application_name', + 'server_address', 'auth_token', 'enable_logging', 'sample_rate', 'detect_subprocesses','oncpu', 'native', 'gil_only')) def configure(app_name=None, application_name=None, server_address="http://localhost:4040", - auth_token = "", sample_rate=100, detect_subprocesses=False, + auth_token="", enable_logging=False, sample_rate=100, detect_subprocesses=False, oncpu=True, native=False, gil_only=True, report_pid=False, report_thread_id=False, report_thread_name=False, tags=None): @@ -26,9 +27,10 @@ def configure(app_name=None, application_name=None, server_address="http://local # INFO = 20 # DEBUG = 10 # NOTSET = 0 - logger = logging.getLogger() - log_level = logger.getEffectiveLevel() - lib.initialize_logging(log_level) + if enable_logging: + logger = logging.getLogger() + log_level = logger.getEffectiveLevel() + lib.initialize_logging(log_level) # Initialize Pyroscope Agent lib.initialize_agent(application_name.encode("UTF-8"), From 81e38c97e867fedeb3e346a859ae1514ead62870 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 00:52:22 -0500 Subject: [PATCH 12/70] imp(ruby): add logging support --- pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 64 +++++++++++++++++-------- pyroscope_ffi/ruby/lib/pyroscope.rb | 26 +++++++++- 2 files changed, 68 insertions(+), 22 deletions(-) diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index 6f045a66..ee6c5c4f 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -3,10 +3,10 @@ use pyroscope::backend::{Report, StackFrame, Tag}; use pyroscope::PyroscopeAgent; use pyroscope_rbspy::{rbspy_backend, RbspyConfig}; use std::collections::hash_map::DefaultHasher; +use std::env; use std::ffi::CStr; use std::hash::Hasher; use std::os::raw::c_char; -use std::env; pub fn transform_report(report: Report) -> Report { let cwd = env::current_dir().unwrap(); @@ -24,31 +24,25 @@ pub fn transform_report(report: Report) -> Report { let mut s = frame.filename.unwrap(); match s.find(cwd) { Some(i) => { - s = s[(i+cwd.len()+1)..].to_string(); + s = s[(i + cwd.len() + 1)..].to_string(); } - None => { - match s.find("/gems/") { + None => match s.find("/gems/") { + Some(i) => { + s = s[(i + 1)..].to_string(); + } + None => match s.find("/ruby/") { Some(i) => { - s = s[(i+1)..].to_string(); - } - None => { - match s.find("/ruby/") { + s = s[(i + 6)..].to_string(); + match s.find("/") { Some(i) => { - s = s[(i+6)..].to_string(); - match s.find("/") { - Some(i) => { - s = s[(i+1)..].to_string(); - } - None => { - } - } - } - None => { + s = s[(i + 1)..].to_string(); } + None => {} } } - } - } + None => {} + }, + }, } // something @@ -76,6 +70,36 @@ pub fn transform_report(report: Report) -> Report { new_report } +#[no_mangle] +pub extern "C" fn initialize_logging(logging_level: u32) -> bool { + // Force rustc to display the log messages in the console. + match logging_level { + 50 => { + std::env::set_var("RUST_LOG", "error"); + } + 40 => { + std::env::set_var("RUST_LOG", "warn"); + } + 30 => { + std::env::set_var("RUST_LOG", "info"); + } + 20 => { + std::env::set_var("RUST_LOG", "debug"); + } + 10 => { + std::env::set_var("RUST_LOG", "trace"); + } + _ => { + std::env::set_var("RUST_LOG", "debug"); + } + } + + // Initialize the logger. + pretty_env_logger::init_timed(); + + true +} + #[no_mangle] pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index 11929703..e854cf3a 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -4,6 +4,7 @@ module Pyroscope module Rust extend FFI::Library ffi_lib File.expand_path(File.dirname(__FILE__)) + "/rbspy/rbspy.#{RbConfig::CONFIG["DLEXT"]}" + attach_function :initialize_logging, [:int], :bool attach_function :initialize_agent, [:string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool attach_function :add_thread_tag, [:uint64, :string, :string], :bool attach_function :remove_thread_tag, [:uint64, :string, :string], :bool @@ -18,7 +19,7 @@ module Utils attach_function :thread_id, [], :uint64 end - Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :log_level, :tags) do + Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :log_level, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :tags) do def initialize(*) self.application_name = '' self.server_address = 'http://localhost:4040' @@ -28,7 +29,7 @@ def initialize(*) self.on_cpu = true self.report_pid = false self.report_thread_id = false - self.log_level = 'info' + self.log_level = 'error' self.tags = {} super end @@ -41,6 +42,27 @@ def configure # Pass config to the block yield @config + # Determine Logging level (kinda like an enum). + case @config.log_level + when 'trace' + @log_level = 10 + when 'debug' + @log_level = 20 + when 'info' + @log_level = 30 + when 'warn' + @log_level = 40 + when 'error' + @log_level = 50 + else + @log_level = 50 + end + + # Initialize Logging + Rust.initialize_logging(@log_level) + + + # initialize Pyroscope Agent Rust.initialize_agent( @config.app_name || @config.application_name || "", @config.server_address || "", From 827dedf402425f3e84edaecd542c8243e5a7eeb4 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 01:14:37 -0500 Subject: [PATCH 13/70] refactor(ruby): update ruby spec --- README.md | 4 ++-- pyroscope_ffi/ruby/pyroscope.gemspec | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cc0e6af3..9815a3c7 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ The Pyroscope Agent doesn't do any profiling. The agent role is to orchasrate a Pyroscope can be used directly from your projects with native integration. No agent or external programs are required. -- [Python](https://pypi.org/project/pyroscope-io/): Python Package. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/python#readme)|[Documentation](https://pyroscope.io/docs/python/) -- [Ruby](https://rubygems.org/gems/pyroscope): Ruby Gem. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/ruby#readme)|[Documentation](https://pyroscope.io/docs/ruby/) +- [Python](https://pypi.org/project/pyroscope-io/): Python Package. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/python#readme) - [Documentation](https://pyroscope.io/docs/python/) +- [Ruby](https://rubygems.org/gems/pyroscope): Ruby Gem. [Readme](https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/ruby#readme) - [Documentation](https://pyroscope.io/docs/ruby/) ### Limitations diff --git a/pyroscope_ffi/ruby/pyroscope.gemspec b/pyroscope_ffi/ruby/pyroscope.gemspec index 8b480326..2cfebb9b 100644 --- a/pyroscope_ffi/ruby/pyroscope.gemspec +++ b/pyroscope_ffi/ruby/pyroscope.gemspec @@ -1,4 +1,11 @@ -require_relative "lib/pyroscope/version" +# coding: utf-8 +# frozen_string_literal: true + +begin + require File.expand_path(File.join(File.dirname(__FILE__), "lib/pyroscope/version")) +rescue LoadError + puts "WARNING: Could not load Pyroscope::VERSION" +end Gem::Specification.new do |s| s.name = 'pyroscope' @@ -9,6 +16,13 @@ Gem::Specification.new do |s| s.email = ['contact@pyroscope.io'] s.homepage = 'https://pyroscope.io' s.license = 'Apache-2.0' + s.metadata = { + "homepage_uri" => "https://pyroscope.io", + "bug_tracker_uri" => "https://github.com/pyroscope-io/pyroscope-rs/issues", + "documentation_uri" => "https://pyroscope.io/docs/ruby/", + "changelog_uri" => "https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/ruby/CHANGELOG.md", + "source_code_uri" => "https://github.com/pyroscope-io/pyroscope-rs/tree/main/pyroscope_ffi/ruby", + } # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. @@ -21,7 +35,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY - s.required_ruby_version = ">= 2.5.9" + s.required_ruby_version = ">= 1.9.3" s.extensions = ['ext/rbspy/extconf.rb', 'ext/thread_id/extconf.rb'] From 62a4554089f61df37587601e62de87896ce4b2b3 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 12:51:30 -0500 Subject: [PATCH 14/70] chore(actions): renamed ruby/python ci files --- .github/workflows/{python.yml => ci-ffi-python.yml} | 0 .github/workflows/{ruby.yml => ci-ffi-ruby.yml} | 0 pyroscope_ffi/ruby/lib/pyroscope.rb | 3 +++ 3 files changed, 3 insertions(+) rename .github/workflows/{python.yml => ci-ffi-python.yml} (100%) rename .github/workflows/{ruby.yml => ci-ffi-ruby.yml} (100%) diff --git a/.github/workflows/python.yml b/.github/workflows/ci-ffi-python.yml similarity index 100% rename from .github/workflows/python.yml rename to .github/workflows/ci-ffi-python.yml diff --git a/.github/workflows/ruby.yml b/.github/workflows/ci-ffi-ruby.yml similarity index 100% rename from .github/workflows/ruby.yml rename to .github/workflows/ci-ffi-ruby.yml diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index e854cf3a..076c5f78 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -1,3 +1,6 @@ +# coding: utf-8 +# frozen_string_literal: true + require 'ffi' module Pyroscope From 2920ab871133de0fb246457420abbf3c46bc74c0 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 13:07:16 -0500 Subject: [PATCH 15/70] wip: update action path --- .github/workflows/ci-ffi-python.yml | 4 ++-- .github/workflows/ci-ffi-ruby.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 30aa78a5..3a0c6754 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -4,13 +4,13 @@ on: push: paths: - 'pyroscope_ffi/python/**' - - '.github/workflows/python.yml' + - '.github/workflows/ci-ffi-python.yml' branches: - '**' pull_request: paths: - 'pyroscope_ffi/python/**' - - '.github/workflows/python.yml' + - '.github/workflows/ci-ffi-python.yml' branches: - '**' diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 15898f53..1d6c6666 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -4,13 +4,13 @@ on: push: paths: - 'pyroscope_ffi/ruby/**' - - '.github/workflows/ruby.yml' + - '.github/workflows/ci-ffi-ruby.yml' branches: - '**' pull_request: paths: - 'pyroscope_ffi/ruby/**' - - '.github/workflows/ruby.yml' + - '.github/workflows/ci-ffi-ruby.yml' branches: - '**' From c47dd49f963771f15016019bb17bd1c9b51f9636 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 13:29:52 -0500 Subject: [PATCH 16/70] wip: test download artifact --- .github/workflows/ci-ffi-python.yml | 184 +++++++++++++++------------- 1 file changed, 98 insertions(+), 86 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 3a0c6754..ee23210a 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -15,7 +15,7 @@ on: - '**' jobs: - linux: + linux-build: strategy: fail-fast: false matrix: @@ -43,94 +43,106 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: ${{ github.sha }} + name: "linux.whl" path: pyroscope_ffi/python/dist/* - linux-arm: - strategy: - fail-fast: false - matrix: - build-arch: - - manylinux2014_aarch64 - - name: Linux - ${{ matrix.build-arch }} - runs-on: [self-hosted, Linux, ARM64] - - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Build in Docker - run: pyroscope_ffi/python/scripts/docker.sh - env: - BUILD_ARCH: ${{ matrix.build-arch }} - - - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }} - path: pyroscope_ffi/python/dist/* - - macos: - strategy: - fail-fast: false - matrix: - include: - - macos-version: "10.15" - target: x86_64-apple-darwin - py-platform: macosx-10_15_x86_64 - - macos-version: "11.0" - target: aarch64-apple-darwin - py-platform: macosx-11_0_arm64 - - name: macOS - ${{ matrix.py-platform }} - runs-on: macos-${{ matrix.macos-version }} - - steps: - - uses: actions/checkout@v2 - - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: ${{ matrix.target }} - profile: minimal - override: true - - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - - name: Build Wheel - run: | - pip install wheel - python setup.py bdist_wheel -p ${{ matrix.py-platform }} - working-directory: pyroscope_ffi/python - env: - CARGO_BUILD_TARGET: ${{ matrix.target }} - - - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }} - path: pyroscope_ffi/python/dist/* - - sdist: - name: sdist + linux-test: + name: Linux Test runs-on: ubuntu-latest steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Build sdist - run: python setup.py sdist - working-directory: pyroscope_ffi/python - - - uses: actions/upload-artifact@v2 + - uses: actions/download-artifact@master with: - name: ${{ github.sha }} - path: pyroscope_ffi/python/dist/* + name: "linux.whl" + path: /python + + - run: cd /python && ls -l + + #linux-arm: + #strategy: + #fail-fast: false + #matrix: + #build-arch: + #- manylinux2014_aarch64 + + #name: Linux - ${{ matrix.build-arch }} + #runs-on: [self-hosted, Linux, ARM64] + + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 + #with: + #submodules: recursive + + #- name: Build in Docker + #run: pyroscope_ffi/python/scripts/docker.sh + #env: + #BUILD_ARCH: ${{ matrix.build-arch }} + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/python/dist/* + + #macos: + #strategy: + #fail-fast: false + #matrix: + #include: + #- macos-version: "10.15" + #target: x86_64-apple-darwin + #py-platform: macosx-10_15_x86_64 + #- macos-version: "11.0" + #target: aarch64-apple-darwin + #py-platform: macosx-11_0_arm64 + + #name: macOS - ${{ matrix.py-platform }} + #runs-on: macos-${{ matrix.macos-version }} + + #steps: + #- uses: actions/checkout@v2 + + #- uses: actions-rs/toolchain@v1 + #with: + #toolchain: stable + #target: ${{ matrix.target }} + #profile: minimal + #override: true + + #- uses: actions/setup-python@v2 + #with: + #python-version: 3.9 + + #- name: Build Wheel + #run: | + #pip install wheel + #python setup.py bdist_wheel -p ${{ matrix.py-platform }} + #working-directory: pyroscope_ffi/python + #env: + #CARGO_BUILD_TARGET: ${{ matrix.target }} + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/python/dist/* + + #sdist: + #name: sdist + #runs-on: ubuntu-latest + + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 + + #- uses: actions/setup-python@v2 + #with: + #python-version: 3.7 + + #- name: Build sdist + #run: python setup.py sdist + #working-directory: pyroscope_ffi/python + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/python/dist/* From bf18b980084b279b2e13d8536da5198193bfed29 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 13:31:06 -0500 Subject: [PATCH 17/70] wip: fix --- .github/workflows/ci-ffi-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index ee23210a..54bdd7d3 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -47,6 +47,7 @@ jobs: path: pyroscope_ffi/python/dist/* linux-test: + needs: ['linux'] name: Linux Test runs-on: ubuntu-latest From fa0941460e9f379c44a543c99d76b973f4cffc5c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 13:32:06 -0500 Subject: [PATCH 18/70] wip: fix --- .github/workflows/ci-ffi-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 54bdd7d3..d0ea8349 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -47,7 +47,7 @@ jobs: path: pyroscope_ffi/python/dist/* linux-test: - needs: ['linux'] + needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest From a6471790f8f7c1c5644f9a943dd2a513c78c9f2d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 19:05:36 -0500 Subject: [PATCH 19/70] wip: fix --- .github/workflows/ci-ffi-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index d0ea8349..6d588992 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -55,9 +55,9 @@ jobs: - uses: actions/download-artifact@master with: name: "linux.whl" - path: /python + path: ${{ GITHUB_WORKSPACE }}/python - - run: cd /python && ls -l + - run: cd ${{ GITHUB_WORKSPACE }}/python && ls -l #linux-arm: #strategy: From 364b1a28909b128f766675ed30df8f95133f0c35 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 19:08:25 -0500 Subject: [PATCH 20/70] wip: use env --- .github/workflows/ci-ffi-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 6d588992..acaaebb2 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -55,9 +55,9 @@ jobs: - uses: actions/download-artifact@master with: name: "linux.whl" - path: ${{ GITHUB_WORKSPACE }}/python + path: "${{env.GITHUB_WORKSPACE}}/python" - - run: cd ${{ GITHUB_WORKSPACE }}/python && ls -l + - run: "cd ${{env.GITHUB_WORKSPACE}}/python && ls -l" #linux-arm: #strategy: From 5a1193c29bf371dd2d1194a18aee9dd062982be2 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 19:21:15 -0500 Subject: [PATCH 21/70] wip: use github.workspace --- .github/workflows/ci-ffi-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index acaaebb2..2f810340 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -55,9 +55,9 @@ jobs: - uses: actions/download-artifact@master with: name: "linux.whl" - path: "${{env.GITHUB_WORKSPACE}}/python" + path: "${{github.workspace}}/python" - - run: "cd ${{env.GITHUB_WORKSPACE}}/python && ls -l" + - run: "cd ${{ github.workspace }}/python && ls -l" #linux-arm: #strategy: From ff1676c1b8adec636981145ae4c9e0f63323b3b2 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 21:17:43 -0500 Subject: [PATCH 22/70] wip: pip install --- .github/workflows/ci-ffi-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 2f810340..bfa202fe 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -58,6 +58,7 @@ jobs: path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" + - run: "pip install *.whl" #linux-arm: #strategy: From 3c29ec3c69cca504f9ccb7c0a386437996cd5a55 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 21:19:34 -0500 Subject: [PATCH 23/70] wip: sdist job --- .github/workflows/ci-ffi-python.yml | 46 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index bfa202fe..8af5b9f8 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -128,23 +128,37 @@ jobs: #name: ${{ github.sha }} #path: pyroscope_ffi/python/dist/* - #sdist: - #name: sdist - #runs-on: ubuntu-latest + sdist-build: + name: sdist + runs-on: ubuntu-latest - #steps: - #- uses: AutoModality/action-clean@v1 - #- uses: actions/checkout@v2 + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 - #- uses: actions/setup-python@v2 - #with: - #python-version: 3.7 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 - #- name: Build sdist - #run: python setup.py sdist - #working-directory: pyroscope_ffi/python + - name: Build sdist + run: python setup.py sdist + working-directory: pyroscope_ffi/python - #- uses: actions/upload-artifact@v2 - #with: - #name: ${{ github.sha }} - #path: pyroscope_ffi/python/dist/* + - uses: actions/upload-artifact@v2 + with: + name: "sdist.whl" + path: pyroscope_ffi/python/dist/* + + sdist-test: + needs: ['sdist-build'] + name: Linux Test + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@master + with: + name: "sdist.whl" + path: "${{github.workspace}}/python" + + - run: "cd ${{ github.workspace }}/python && ls -l" + - run: "pip install *.whl" From 660e2aff81a7bec1a82735f1a7e8828fd97eb964 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 21:26:06 -0500 Subject: [PATCH 24/70] wip: cd into directory --- .github/workflows/ci-ffi-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 8af5b9f8..37d04365 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -161,4 +161,4 @@ jobs: path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "pip install *.whl" + - run: "cd ${{ github.workspace}}/python && pip install *.whl" From 28c021e97f1997125be938869a7befbc091fb9e5 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Thu, 28 Jul 2022 21:29:57 -0500 Subject: [PATCH 25/70] wip: wrong place --- .github/workflows/ci-ffi-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 37d04365..1a272d7e 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -58,7 +58,7 @@ jobs: path: "${{github.workspace}}/python" - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "pip install *.whl" + - run: "cd ${{ github.workspace }}/python && pip install *.whl" #linux-arm: #strategy: From 8b9b96cb37d25da08d5c13a27a2f9cf4fa538131 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 00:15:29 -0500 Subject: [PATCH 26/70] wip: python test files --- pyroscope_ffi/python/scripts/tests/all.py | 54 +++++++++++++++++++ pyroscope_ffi/python/scripts/tests/gil.py | 53 ++++++++++++++++++ .../python/scripts/tests/oncpu-gil.py | 53 ++++++++++++++++++ pyroscope_ffi/python/scripts/tests/oncpu.py | 53 ++++++++++++++++++ pyroscope_ffi/python/scripts/tests/sub-gil.py | 53 ++++++++++++++++++ .../python/scripts/tests/sub-oncpu.py | 53 ++++++++++++++++++ pyroscope_ffi/python/scripts/tests/sub.py | 53 ++++++++++++++++++ 7 files changed, 372 insertions(+) create mode 100755 pyroscope_ffi/python/scripts/tests/all.py create mode 100755 pyroscope_ffi/python/scripts/tests/gil.py create mode 100755 pyroscope_ffi/python/scripts/tests/oncpu-gil.py create mode 100755 pyroscope_ffi/python/scripts/tests/oncpu.py create mode 100755 pyroscope_ffi/python/scripts/tests/sub-gil.py create mode 100755 pyroscope_ffi/python/scripts/tests/sub-oncpu.py create mode 100755 pyroscope_ffi/python/scripts/tests/sub.py diff --git a/pyroscope_ffi/python/scripts/tests/all.py b/pyroscope_ffi/python/scripts/tests/all.py new file mode 100755 index 00000000..e0a6cea2 --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/all.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), + enable_logging=True, + detect_subprocesses=True, + oncpu=True, + gil_only=True, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/gil.py b/pyroscope_ffi/python/scripts/tests/gil.py new file mode 100755 index 00000000..68449d91 --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/gil.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=False, + oncpu=False, + gil_only=True, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py new file mode 100755 index 00000000..ff37bc7c --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=False, + oncpu=True, + gil_only=True, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/oncpu.py b/pyroscope_ffi/python/scripts/tests/oncpu.py new file mode 100755 index 00000000..3e1d16a8 --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/oncpu.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=False, + oncpu=True, + gil_only=False, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub-gil.py b/pyroscope_ffi/python/scripts/tests/sub-gil.py new file mode 100755 index 00000000..848dae0f --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/sub-gil.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=True, + oncpu=False, + gil_only=True, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py new file mode 100755 index 00000000..279c8693 --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=True, + oncpu=True, + gil_only=False, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub.py b/pyroscope_ffi/python/scripts/tests/sub.py new file mode 100755 index 00000000..eb31001c --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/sub.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + app_name = "run.python.app", + server_address = "http://localhost:4040", + enable_logging=True, + detect_subprocesses=True, + oncpu=False, + gil_only=False, + report_pid=True, + report_thread_id=True, + report_thread_name=True, +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 55510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 55510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() From d32ef81845a3d513d6f18c32ae6a2162f7bcb82d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 00:19:28 -0500 Subject: [PATCH 27/70] wip: add python script runner --- .github/workflows/ci-ffi-python.yml | 62 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 1a272d7e..46d60be9 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -50,7 +50,6 @@ jobs: needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest - steps: - uses: actions/download-artifact@master with: @@ -59,6 +58,13 @@ jobs: - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip install *.whl" + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/all.py + env: + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} #linux-arm: #strategy: @@ -128,37 +134,37 @@ jobs: #name: ${{ github.sha }} #path: pyroscope_ffi/python/dist/* - sdist-build: - name: sdist - runs-on: ubuntu-latest + #sdist-build: + #name: sdist + #runs-on: ubuntu-latest - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.7 + #- uses: actions/setup-python@v2 + #with: + #python-version: 3.7 - - name: Build sdist - run: python setup.py sdist - working-directory: pyroscope_ffi/python + #- name: Build sdist + #run: python setup.py sdist + #working-directory: pyroscope_ffi/python - - uses: actions/upload-artifact@v2 - with: - name: "sdist.whl" - path: pyroscope_ffi/python/dist/* + #- uses: actions/upload-artifact@v2 + #with: + #name: "sdist.whl" + #path: pyroscope_ffi/python/dist/* - sdist-test: - needs: ['sdist-build'] - name: Linux Test - runs-on: ubuntu-latest + #sdist-test: + #needs: ['sdist-build'] + #name: Linux Test + #runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@master - with: - name: "sdist.whl" - path: "${{github.workspace}}/python" + #steps: + #- uses: actions/download-artifact@master + #with: + #name: "sdist.whl" + #path: "${{github.workspace}}/python" - - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "cd ${{ github.workspace}}/python && pip install *.whl" + #- run: "cd ${{ github.workspace }}/python && ls -l" + #- run: "cd ${{ github.workspace}}/python && pip install *.whl" From 91343129d9693cb831ab4765cdfe7f7e632b5d4b Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 00:38:04 -0500 Subject: [PATCH 28/70] wip: add run_id --- .github/workflows/ci-ffi-python.yml | 1 + pyroscope_ffi/python/scripts/tests/all.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 46d60be9..41e7e537 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -64,6 +64,7 @@ jobs: - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/all.py env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} #linux-arm: diff --git a/pyroscope_ffi/python/scripts/tests/all.py b/pyroscope_ffi/python/scripts/tests/all.py index e0a6cea2..bb9e760e 100755 --- a/pyroscope_ffi/python/scripts/tests/all.py +++ b/pyroscope_ffi/python/scripts/tests/all.py @@ -13,10 +13,10 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, + enable_logging =True, detect_subprocesses=True, oncpu=True, gil_only=True, From 4c48be9ee5b959d4ba545df01f74958ccc334efa Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 01:02:28 -0500 Subject: [PATCH 29/70] wip: add other scripts --- .github/workflows/ci-ffi-python.yml | 31 +++++++++++++++++++ pyroscope_ffi/python/scripts/tests/all.py | 6 ++-- pyroscope_ffi/python/scripts/tests/gil.py | 9 +++--- .../python/scripts/tests/oncpu-gil.py | 10 +++--- pyroscope_ffi/python/scripts/tests/oncpu.py | 9 +++--- pyroscope_ffi/python/scripts/tests/sub-gil.py | 10 +++--- .../python/scripts/tests/sub-oncpu.py | 10 +++--- pyroscope_ffi/python/scripts/tests/sub.py | 9 +++--- 8 files changed, 67 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 41e7e537..08db5659 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -66,6 +66,37 @@ jobs: env: PYROSCOPE_RUN_ID: ${{ github.run_id }} PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + #linux-arm: #strategy: diff --git a/pyroscope_ffi/python/scripts/tests/all.py b/pyroscope_ffi/python/scripts/tests/all.py index bb9e760e..b797defb 100755 --- a/pyroscope_ffi/python/scripts/tests/all.py +++ b/pyroscope_ffi/python/scripts/tests/all.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-all', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging =True, @@ -33,12 +33,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/gil.py b/pyroscope_ffi/python/scripts/tests/gil.py index 68449d91..31608d85 100755 --- a/pyroscope_ffi/python/scripts/tests/gil.py +++ b/pyroscope_ffi/python/scripts/tests/gil.py @@ -13,8 +13,9 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-gil', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=False, oncpu=False, @@ -32,12 +33,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py index ff37bc7c..038ce7e3 100755 --- a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py +++ b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py @@ -13,8 +13,10 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = + f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-oncpu-gil', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=False, oncpu=True, @@ -32,12 +34,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/oncpu.py b/pyroscope_ffi/python/scripts/tests/oncpu.py index 3e1d16a8..0028f350 100755 --- a/pyroscope_ffi/python/scripts/tests/oncpu.py +++ b/pyroscope_ffi/python/scripts/tests/oncpu.py @@ -13,8 +13,9 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-onpcu', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=False, oncpu=True, @@ -32,12 +33,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/sub-gil.py b/pyroscope_ffi/python/scripts/tests/sub-gil.py index 848dae0f..52772d1a 100755 --- a/pyroscope_ffi/python/scripts/tests/sub-gil.py +++ b/pyroscope_ffi/python/scripts/tests/sub-gil.py @@ -13,8 +13,10 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = + f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub:gil', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=True, oncpu=False, @@ -32,12 +34,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py index 279c8693..02d173fe 100755 --- a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py +++ b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py @@ -13,8 +13,10 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = + f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub:oncpu', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=True, oncpu=True, @@ -32,12 +34,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string diff --git a/pyroscope_ffi/python/scripts/tests/sub.py b/pyroscope_ffi/python/scripts/tests/sub.py index eb31001c..e6a6c456 100755 --- a/pyroscope_ffi/python/scripts/tests/sub.py +++ b/pyroscope_ffi/python/scripts/tests/sub.py @@ -13,8 +13,9 @@ # Configure Pyroscope pyroscope.configure( - app_name = "run.python.app", - server_address = "http://localhost:4040", + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, detect_subprocesses=True, oncpu=False, @@ -32,12 +33,12 @@ def hash(string): return string def multihash(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string def multihash2(string): - for i in range(0, 55510055): + for i in range(0, 25510055): string = hash(string) return string From 0c91a61718a967a425c5e9b04d8f2534314fffaf Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 15:26:40 -0500 Subject: [PATCH 30/70] wip: arm tests --- .github/workflows/ci-ffi-python.yml | 115 ++++++++++++++---- pyroscope_ffi/python/scripts/tests/all.py | 2 +- pyroscope_ffi/python/scripts/tests/gil.py | 2 +- .../python/scripts/tests/oncpu-gil.py | 3 +- pyroscope_ffi/python/scripts/tests/oncpu.py | 2 +- pyroscope_ffi/python/scripts/tests/sub-gil.py | 3 +- .../python/scripts/tests/sub-oncpu.py | 3 +- pyroscope_ffi/python/scripts/tests/sub.py | 2 +- 8 files changed, 97 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 08db5659..7d4b3389 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -65,73 +65,138 @@ jobs: run: pyroscope_ffi/python/scripts/tests/all.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/gil.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/oncpu.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/sub-gil.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - name: Run Python Script run: pyroscope_ffi/python/scripts/tests/sub.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - #linux-arm: - #strategy: - #fail-fast: false - #matrix: - #build-arch: - #- manylinux2014_aarch64 + linux-arm-build: + strategy: + fail-fast: false + matrix: + build-arch: + - manylinux2014_aarch64 - #name: Linux - ${{ matrix.build-arch }} - #runs-on: [self-hosted, Linux, ARM64] + name: Linux - ${{ matrix.build-arch }} + runs-on: [self-hosted, Linux, ARM64] - #steps: - #- uses: AutoModality/action-clean@v1 - #- uses: actions/checkout@v2 - #with: - #submodules: recursive + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 + with: + submodules: recursive - #- name: Build in Docker - #run: pyroscope_ffi/python/scripts/docker.sh - #env: - #BUILD_ARCH: ${{ matrix.build-arch }} + - name: Build in Docker + run: pyroscope_ffi/python/scripts/docker.sh + env: + BUILD_ARCH: ${{ matrix.build-arch }} - #- uses: actions/upload-artifact@v2 - #with: - #name: ${{ github.sha }} - #path: pyroscope_ffi/python/dist/* + - uses: actions/upload-artifact@v2 + with: + name: "linux-arm.whl" + path: pyroscope_ffi/python/dist/* - #macos: + linux-arm-test: + needs: ['linux-arm-build'] + name: Linux ARM Test + runs-on: [self-hosted, Linux, ARM64] + steps: + - uses: actions/download-artifact@master + with: + name: "linux-arm.whl" + path: "${{github.workspace}}/python" + + - run: "cd ${{ github.workspace }}/python && ls -l" + - run: "cd ${{ github.workspace }}/python && pip install *.whl" + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/all.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: aarch64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + + #macos-build: #strategy: #fail-fast: false #matrix: #include: - #- macos-version: "10.15" + #- macos-version: "11.0" #target: x86_64-apple-darwin - #py-platform: macosx-10_15_x86_64 + #py-platform: macosx-11_0_x86_64 #- macos-version: "11.0" #target: aarch64-apple-darwin #py-platform: macosx-11_0_arm64 @@ -163,7 +228,7 @@ jobs: #- uses: actions/upload-artifact@v2 #with: - #name: ${{ github.sha }} + #name: ${{ matrix.target }} #path: pyroscope_ffi/python/dist/* #sdist-build: diff --git a/pyroscope_ffi/python/scripts/tests/all.py b/pyroscope_ffi/python/scripts/tests/all.py index b797defb..46197d85 100755 --- a/pyroscope_ffi/python/scripts/tests/all.py +++ b/pyroscope_ffi/python/scripts/tests/all.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-all', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-all', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging =True, diff --git a/pyroscope_ffi/python/scripts/tests/gil.py b/pyroscope_ffi/python/scripts/tests/gil.py index 31608d85..4e6049ef 100755 --- a/pyroscope_ffi/python/scripts/tests/gil.py +++ b/pyroscope_ffi/python/scripts/tests/gil.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-gil', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-gil', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, diff --git a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py index 038ce7e3..eb510af1 100755 --- a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py +++ b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py @@ -13,8 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = - f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-oncpu-gil', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-oncpu:gil', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, diff --git a/pyroscope_ffi/python/scripts/tests/oncpu.py b/pyroscope_ffi/python/scripts/tests/oncpu.py index 0028f350..33945ca6 100755 --- a/pyroscope_ffi/python/scripts/tests/oncpu.py +++ b/pyroscope_ffi/python/scripts/tests/oncpu.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-onpcu', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-onpcu', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, diff --git a/pyroscope_ffi/python/scripts/tests/sub-gil.py b/pyroscope_ffi/python/scripts/tests/sub-gil.py index 52772d1a..fd99fc83 100755 --- a/pyroscope_ffi/python/scripts/tests/sub-gil.py +++ b/pyroscope_ffi/python/scripts/tests/sub-gil.py @@ -13,8 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = - f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub:gil', + application_name =f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub:gil', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, diff --git a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py index 02d173fe..91c0d3b1 100755 --- a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py +++ b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py @@ -13,8 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = - f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub:oncpu', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub:oncpu', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, diff --git a/pyroscope_ffi/python/scripts/tests/sub.py b/pyroscope_ffi/python/scripts/tests/sub.py index e6a6c456..1b7ee40b 100755 --- a/pyroscope_ffi/python/scripts/tests/sub.py +++ b/pyroscope_ffi/python/scripts/tests/sub.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-x86-64-linux-sub', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub', server_address = "https://ingest.pyroscope.cloud", auth_token = os.getenv("PYROSCOPE_API_TOKEN"), enable_logging=True, From 6016b7d56280ef06928fe94b12d40b7998db9d90 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:12:56 -0500 Subject: [PATCH 31/70] wip: add setup_python --- .github/workflows/ci-ffi-python.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 7d4b3389..47a03984 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -136,6 +136,9 @@ jobs: name: Linux ARM Test runs-on: [self-hosted, Linux, ARM64] steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.x' - uses: actions/download-artifact@master with: name: "linux-arm.whl" From b99c840778b0703a0a70c745d3f1cbe9632a58a7 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:21:28 -0500 Subject: [PATCH 32/70] wip: add macos tests --- .github/workflows/ci-ffi-python.yml | 131 ++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 47a03984..4236e0fd 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -192,47 +192,106 @@ jobs: PYROSCOPE_ARCH: aarch64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - #macos-build: - #strategy: - #fail-fast: false - #matrix: - #include: - #- macos-version: "11.0" - #target: x86_64-apple-darwin - #py-platform: macosx-11_0_x86_64 - #- macos-version: "11.0" - #target: aarch64-apple-darwin - #py-platform: macosx-11_0_arm64 - - #name: macOS - ${{ matrix.py-platform }} - #runs-on: macos-${{ matrix.macos-version }} + macos-build: + strategy: + fail-fast: false + matrix: + include: + - macos-version: "11.0" + target: x86_64-apple-darwin + py-platform: macosx-11_0_x86_64 + - macos-version: "11.0" + target: aarch64-apple-darwin + py-platform: macosx-11_0_arm64 - #steps: - #- uses: actions/checkout@v2 + name: macOS - ${{ matrix.py-platform }} + runs-on: macos-${{ matrix.macos-version }} - #- uses: actions-rs/toolchain@v1 - #with: - #toolchain: stable - #target: ${{ matrix.target }} - #profile: minimal - #override: true + steps: + - uses: actions/checkout@v2 - #- uses: actions/setup-python@v2 - #with: - #python-version: 3.9 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: ${{ matrix.target }} + profile: minimal + override: true - #- name: Build Wheel - #run: | - #pip install wheel - #python setup.py bdist_wheel -p ${{ matrix.py-platform }} - #working-directory: pyroscope_ffi/python - #env: - #CARGO_BUILD_TARGET: ${{ matrix.target }} + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Build Wheel + run: | + pip install wheel + python setup.py bdist_wheel -p ${{ matrix.py-platform }} + working-directory: pyroscope_ffi/python + env: + CARGO_BUILD_TARGET: ${{ matrix.target }} + + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.target }} + path: pyroscope_ffi/python/dist/* + + macos-test: + needs: ['macos-build'] + name: Macos Test + runs-on: macos-11.0 + steps: + - uses: actions/download-artifact@master + with: + name: x86_64-apple-darwin + path: "${{github.workspace}}/python" + + - run: "cd ${{ github.workspace }}/python && ls -l" + - run: "cd ${{ github.workspace }}/python && pip install *.whl" + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/all.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-gil.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/sub.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - #- uses: actions/upload-artifact@v2 - #with: - #name: ${{ matrix.target }} - #path: pyroscope_ffi/python/dist/* #sdist-build: #name: sdist From 866baad13702c238de77295f0ded5e03c6085ac3 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:26:02 -0500 Subject: [PATCH 33/70] wip: install pip for arm --- .github/workflows/ci-ffi-python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 4236e0fd..60704cac 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -144,6 +144,8 @@ jobs: name: "linux-arm.whl" path: "${{github.workspace}}/python" + - run: "apt update && apt install python3 python3-pip" + - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip install *.whl" - uses: actions/checkout@v2 From 322b8ab3fa6bb89af6430fb0e8d863960c941d72 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:38:50 -0500 Subject: [PATCH 34/70] wip: add setup_python for macos --- .github/workflows/ci-ffi-python.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 60704cac..205c878a 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -136,9 +136,6 @@ jobs: name: Linux ARM Test runs-on: [self-hosted, Linux, ARM64] steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - uses: actions/download-artifact@master with: name: "linux-arm.whl" @@ -241,6 +238,10 @@ jobs: name: Macos Test runs-on: macos-11.0 steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x - uses: actions/download-artifact@master with: name: x86_64-apple-darwin From 4c99b63a072cd7c3409e3265a009ff60f5155374 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:45:27 -0500 Subject: [PATCH 35/70] wip: use apt-get instead --- .github/workflows/ci-ffi-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 205c878a..8d5f7202 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -141,7 +141,7 @@ jobs: name: "linux-arm.whl" path: "${{github.workspace}}/python" - - run: "apt update && apt install python3 python3-pip" + - run: "apt-get update && apt-get install python3 python3-pip" - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip install *.whl" From a00986ed16c196f48e76efd6a68edf9e4f564929 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 29 Jul 2022 18:48:12 -0500 Subject: [PATCH 36/70] wip: update pip --- .github/workflows/ci-ffi-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 8d5f7202..8cbb3797 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -247,6 +247,7 @@ jobs: name: x86_64-apple-darwin path: "${{github.workspace}}/python" + - run: "pip install --upgrade pip" - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip install *.whl" - uses: actions/checkout@v2 From 5b326fdfc84d6a0cc4d3c7007b2970b2acf52c8c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 30 Jul 2022 21:04:49 -0500 Subject: [PATCH 37/70] wip: minor --- .github/workflows/ci-ffi-python.yml | 54 +++++++++++------------------ pyroscope_ffi/ruby/lib/pyroscope.rb | 2 +- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 8cbb3797..354b372a 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -297,37 +297,23 @@ jobs: PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - #sdist-build: - #name: sdist - #runs-on: ubuntu-latest - - #steps: - #- uses: AutoModality/action-clean@v1 - #- uses: actions/checkout@v2 - - #- uses: actions/setup-python@v2 - #with: - #python-version: 3.7 - - #- name: Build sdist - #run: python setup.py sdist - #working-directory: pyroscope_ffi/python - - #- uses: actions/upload-artifact@v2 - #with: - #name: "sdist.whl" - #path: pyroscope_ffi/python/dist/* - - #sdist-test: - #needs: ['sdist-build'] - #name: Linux Test - #runs-on: ubuntu-latest - - #steps: - #- uses: actions/download-artifact@master - #with: - #name: "sdist.whl" - #path: "${{github.workspace}}/python" - - #- run: "cd ${{ github.workspace }}/python && ls -l" - #- run: "cd ${{ github.workspace}}/python && pip install *.whl" + sdist-build: + name: sdist + runs-on: ubuntu-latest + + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Build sdist + run: python setup.py sdist + working-directory: pyroscope_ffi/python + + - uses: actions/upload-artifact@v2 + with: + name: "sdist.whl" + path: pyroscope_ffi/python/dist/* diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index 076c5f78..a0ee5da9 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -121,7 +121,7 @@ def _remove_tags(thread_id, tags) end end - def drop + def shutdown Rust.drop_agent end end From 828a7b6dba4d3c95f1a2b81ed4c77c716f039e24 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 30 Jul 2022 22:10:43 -0500 Subject: [PATCH 38/70] chore(cli): update dependencies --- pyroscope_cli/CHANGELOG.md | 2 +- pyroscope_cli/Cargo.lock | 433 ++++++++++++++++++------------------- pyroscope_cli/Cargo.toml | 12 +- 3 files changed, 215 insertions(+), 232 deletions(-) diff --git a/pyroscope_cli/CHANGELOG.md b/pyroscope_cli/CHANGELOG.md index e9d85f2f..595f69f0 100644 --- a/pyroscope_cli/CHANGELOG.md +++ b/pyroscope_cli/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.2.0 +# v0.2.1 - Update pyroscope lib to 0.5.2 - update pprof-rs backend to 0.2.1 - update rbspy backend to 0.2.1 diff --git a/pyroscope_cli/Cargo.lock b/pyroscope_cli/Cargo.lock index dec4346b..a23af6f3 100644 --- a/pyroscope_cli/Cargo.lock +++ b/pyroscope_cli/Cargo.lock @@ -7,11 +7,20 @@ name = "addr2line" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "addr2line" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ca9b76e919fd83ccfb509f51b28c333c0e03f2221616e347a129215cec4e4a9" dependencies = [ "cpp_demangle", "fallible-iterator", "gimli", - "object 0.27.1", + "object", "rustc-demangle", "smallvec", ] @@ -53,15 +62,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "arc-swap" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "arrayvec" @@ -111,16 +120,16 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" dependencies = [ - "addr2line", + "addr2line 0.17.0", "cc", "cfg-if", "libc", "miniz_oxide", - "object 0.28.4", + "object", "rustc-demangle", ] @@ -182,15 +191,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.9.1" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "bytemuck" -version = "1.9.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" +checksum = "a5377c8865e74a160d21f29c2d40669f53286db6eab59b88540cbb12ffc8b835" [[package]] name = "byteorder" @@ -200,9 +209,9 @@ checksum = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "cc" @@ -266,16 +275,16 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.12" +version = "3.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" +checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9" dependencies = [ "atty", "bitflags", "clap_derive", "clap_lex", "indexmap", - "lazy_static", + "once_cell", "strsim 0.10.0", "termcolor", "terminal_size", @@ -284,18 +293,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "3.1.4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da92e6facd8d73c22745a5d3cbb59bdf8e46e3235c923e516527d8e81eec14a4" +checksum = "9a4fc4a94ae7c4e7c2e721893fce77ef8da529775e2d727d72de8753fcb2ad2a" dependencies = [ - "clap 3.1.12", + "clap 3.2.16", ] [[package]] name = "clap_derive" -version = "3.1.7" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck", "proc-macro-error", @@ -306,9 +315,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.1.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ "os_str_bytes", ] @@ -333,7 +342,7 @@ dependencies = [ "lazy_static", "nom 5.1.2", "rust-ini", - "serde 1.0.137", + "serde 1.0.140", "serde-hjson", "serde_json", "toml", @@ -342,14 +351,13 @@ dependencies = [ [[package]] name = "console" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" +checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847" dependencies = [ "encode_unicode", "libc", "once_cell", - "regex", "terminal_size", "unicode-width", "winapi", @@ -391,9 +399,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if", "crossbeam-utils", @@ -401,12 +409,12 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ "cfg-if", - "lazy_static", + "once_cell", ] [[package]] @@ -415,7 +423,7 @@ version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b37feaa84e6861e00a1f5e5aa8da3ee56d605c9992d33e082786754828e20865" dependencies = [ - "nix 0.24.1", + "nix 0.24.2", "winapi", ] @@ -426,18 +434,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" dependencies = [ "cfg-if", - "hashbrown 0.12.1", + "hashbrown", "lock_api", "parking_lot_core", ] [[package]] name = "debugid" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid", + "uuid 1.1.2", ] [[package]] @@ -507,9 +515,9 @@ dependencies = [ [[package]] name = "either" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "elf" @@ -612,9 +620,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ "instant", ] @@ -737,20 +745,20 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "gimli" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ "fallible-iterator", "stable_deref_trait", @@ -775,9 +783,9 @@ dependencies = [ [[package]] name = "goblin" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c955ab4e0ad8c843ea653a3d143048b87490d9be56bd7132a435c2407846ac8f" +checksum = "91766b1121940d622933a13e20665857648681816089c9bc2075c4b75a6e4f6b" dependencies = [ "log", "plain", @@ -805,19 +813,13 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "ahash", ] -[[package]] -name = "hashbrown" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" - [[package]] name = "heck" version = "0.4.0" @@ -835,9 +837,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", @@ -875,11 +877,11 @@ checksum = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36" dependencies = [ "backtrace", "os_type", - "serde 1.0.137", + "serde 1.0.140", "serde_derive", "termcolor", "toml", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -899,9 +901,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.19" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -960,12 +962,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -982,23 +984,23 @@ dependencies = [ [[package]] name = "inferno" -version = "0.11.4" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3cb215599901c8f491666421d44ffaed08d6872b4c7ced6f425683b951271e" +checksum = "9709543bd6c25fdc748da2bed0f6855b07b7e93a203ae31332ac2101ab2f4782" dependencies = [ "ahash", "atty", - "clap 3.1.12", + "clap 3.2.16", "crossbeam-channel", "crossbeam-utils", "dashmap", "env_logger 0.9.0", "indexmap", "itoa 1.0.2", - "lazy_static", "log", "num-format", "num_cpus", + "once_cell", "quick-xml", "rgb", "str_stack", @@ -1042,9 +1044,9 @@ checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "js-sys" -version = "0.3.57" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" dependencies = [ "wasm-bindgen", ] @@ -1092,9 +1094,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" +checksum = "da83a57f3f5ba3680950aa3cbc806fc297bc0b289d42e8942ed528ace71b8145" [[package]] name = "libproc" @@ -1119,9 +1121,9 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "lock_api" @@ -1144,11 +1146,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8015d95cb7b2ddd3c0d32ca38283ceb1eea09b4713ee380bceb942d85a244228" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -1162,9 +1164,9 @@ dependencies = [ [[package]] name = "mach2" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c29a677a8db77a45ddd1370ac4fbd5187f827780000202079e28949f1a75cb1" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" dependencies = [ "libc", ] @@ -1199,9 +1201,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" +checksum = "3a79b39c93a7a5a27eeaf9a23b5ff43f1b9e0ad6b1cdd441140ae53c35613fc7" dependencies = [ "libc", ] @@ -1238,9 +1240,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", @@ -1254,7 +1256,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d66043b25d4a6cccb23619d10c19c25304b355a7dccd4a8e11423dd2382146" dependencies = [ - "clap 3.1.12", + "clap 3.2.16", "rand 0.8.5", ] @@ -1291,14 +1293,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ "bitflags", "cfg-if", "libc", - "memoffset", ] [[package]] @@ -1400,34 +1401,25 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" dependencies = [ "flate2", "memchr", ] -[[package]] -name = "object" -version = "0.28.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "openssl" -version = "0.10.40" +version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", "cfg-if", @@ -1457,9 +1449,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.73" +version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ "autocfg", "cc", @@ -1480,9 +1472,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.1.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "os_type" @@ -1554,16 +1546,16 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "pprof" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2324292407eab69d4ace0eed1524fe612ac37c98aa22b0d868355b17fada530" +checksum = "1bba88ee898c63351101af3e60c66c5398c517681ce533eef8caff10ecf11ec1" dependencies = [ "backtrace", "cfg-if", "findshlibs", "libc", "log", - "nix 0.23.1", + "nix 0.24.2", "once_cell", "parking_lot", "smallvec", @@ -1644,9 +1636,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" dependencies = [ "unicode-ident", ] @@ -1695,7 +1687,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7663bd5eace848af4fcf0ffa0b273aaecfbf318819f4b8d0214a9f02f69d9940" dependencies = [ "chrono", - "clap 3.1.12", + "clap 3.2.16", "clap_complete", "console", "cpp_demangle", @@ -1715,7 +1707,7 @@ dependencies = [ "rand_distr", "regex", "remoteprocess", - "serde 1.0.137", + "serde 1.0.140", "serde_derive", "serde_json", "tempfile", @@ -1725,22 +1717,23 @@ dependencies = [ [[package]] name = "pyroscope" -version = "0.5.2" +version = "0.5.3" dependencies = [ "libc", "log", "names", "reqwest", "thiserror", + "url", ] [[package]] name = "pyroscope-cli" -version = "0.2.1" +version = "0.2.2" dependencies = [ "assert_cmd", "better-panic", - "clap 3.1.12", + "clap 3.2.16", "clap_complete", "color-backtrace", "config", @@ -1756,7 +1749,7 @@ dependencies = [ "pyroscope_pprofrs", "pyroscope_pyspy", "pyroscope_rbspy", - "serde 1.0.137", + "serde 1.0.140", "slog", "slog-async", "slog-scope", @@ -1767,7 +1760,7 @@ dependencies = [ [[package]] name = "pyroscope_pprofrs" -version = "0.2.1" +version = "0.2.3" dependencies = [ "pprof", "pyroscope", @@ -1776,7 +1769,7 @@ dependencies = [ [[package]] name = "pyroscope_pyspy" -version = "0.2.1" +version = "0.2.3" dependencies = [ "log", "py-spy", @@ -1786,12 +1779,12 @@ dependencies = [ [[package]] name = "pyroscope_rbspy" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "log", "pyroscope", - "rbspy", + "rbspy-oncpu", "thiserror", ] @@ -1812,9 +1805,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -1888,14 +1881,14 @@ dependencies = [ ] [[package]] -name = "rbspy" +name = "rbspy-oncpu" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbebdf0037de5a92722b0bca2965ce0a76cd5c962225cf355e31ac668ea653f" +checksum = "fdc6110000bf4e0c644e1ba4e86877cc84da7a637e3fa394cecedcaf9728870c" dependencies = [ "anyhow", "chrono", - "clap 3.1.12", + "clap 3.2.16", "ctrlc", "directories", "elf", @@ -1911,7 +1904,7 @@ dependencies = [ "rand 0.8.5", "rbspy-ruby-structs", "remoteprocess", - "serde 1.0.137", + "serde 1.0.140", "serde_derive", "serde_json", "tempdir", @@ -1949,9 +1942,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] @@ -1969,9 +1962,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.6" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -1986,18 +1979,18 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remoteprocess" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f13e22b81d40176f4d7ed41c029fd2c7306b182db9838a44d88aeb97a7c6d6" +checksum = "ab91eb6b1c4c507a08d6fb6e2f38982a1e09923d19023506cba25aefcfc6e46f" dependencies = [ - "addr2line", - "goblin 0.5.1", + "addr2line 0.18.0", + "goblin 0.5.3", "lazy_static", "libc", "libproc 0.12.0", @@ -2005,8 +1998,8 @@ dependencies = [ "mach", "mach_o_sys", "memmap", - "nix 0.24.1", - "object 0.27.1", + "nix 0.24.2", + "object", "proc-maps", "read-process-memory", "regex", @@ -2024,9 +2017,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64", "bytes", @@ -2049,13 +2042,14 @@ dependencies = [ "pin-project-lite", "rustls", "rustls-native-certs", - "rustls-pemfile 0.3.0", - "serde 1.0.137", + "rustls-pemfile", + "serde 1.0.140", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", "tokio-rustls", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -2065,9 +2059,9 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74fdc210d8f24a7dbfedc13b04ba5764f5232754ccebfdf5fff1bad791ccbc6" +checksum = "c3b221de559e4a29df3b957eec92bc0de6bc8eaf6ca9cfed43e5e1d67ff65a34" dependencies = [ "bytemuck", ] @@ -2124,20 +2118,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.0", + "rustls-pemfile", "schannel", "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" -dependencies = [ - "base64", -] - [[package]] name = "rustls-pemfile" version = "1.0.0" @@ -2149,9 +2134,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" [[package]] name = "ryu" @@ -2256,9 +2241,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" dependencies = [ "serde_derive", ] @@ -2277,9 +2262,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" dependencies = [ "proc-macro2", "quote", @@ -2288,13 +2273,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ "itoa 1.0.2", "ryu", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -2306,7 +2291,7 @@ dependencies = [ "form_urlencoded", "itoa 1.0.2", "ryu", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -2327,9 +2312,12 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "slab" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "slog" @@ -2381,14 +2369,14 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.9", + "time 0.3.11", ] [[package]] name = "smallvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" @@ -2438,21 +2426,21 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "symbolic-common" -version = "8.7.3" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c276d5b296f30f0fb30d2128b6be6f9804be7669f1edef46c4d8c017de4c77c3" +checksum = "9ea2ab8b85d27d49d184438b4b77fbd521b385cc9c5c802f60e784f2df25a03d" dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid", + "uuid 1.1.2", ] [[package]] name = "symbolic-demangle" -version = "8.7.3" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b096bd4794e4d05584144dc9536a4fde8844da73f84cdb7652ed78b59e37a218" +checksum = "7939b15a1c62633d1fce17f8a7b668312eaa2d3b117bf4ced5e6e77870c43b6a" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -2461,9 +2449,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.95" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", @@ -2627,9 +2615,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" dependencies = [ "itoa 1.0.2", "libc", @@ -2660,10 +2648,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ + "autocfg", "bytes", "libc", "memchr", @@ -2698,9 +2687,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", @@ -2716,45 +2705,33 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -2771,15 +2748,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] @@ -2823,6 +2800,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2874,9 +2857,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2884,13 +2867,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -2899,9 +2882,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ "cfg-if", "js-sys", @@ -2911,9 +2894,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2921,9 +2904,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ "proc-macro2", "quote", @@ -2934,15 +2917,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" [[package]] name = "web-sys" -version = "0.3.57" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/pyroscope_cli/Cargo.toml b/pyroscope_cli/Cargo.toml index 2779e16f..247c879f 100644 --- a/pyroscope_cli/Cargo.toml +++ b/pyroscope_cli/Cargo.toml @@ -5,7 +5,7 @@ Pyroscope Profiler CLI """ keywords = ["pyroscope", "profiler", "cli"] authors = ["Abid Omar "] -version = "0.2.1" +version = "0.2.2" edition = "2021" license = "Apache-2.0" homepage = "https://pyroscope.io" @@ -19,7 +19,7 @@ human-panic = "1.0.3" pretty_env_logger = "0.4.0" better-panic = "0.3.0" log = "=0.4.16" -clap_complete = "3.1" +clap_complete = "3.2" names= "0.13.0" ctrlc = "3.2.2" duct = "0.13.5" @@ -34,12 +34,12 @@ slog-scope = "4.4.0" slog-async = "2.7.0" slog-stdlog = "4.1.0" pyroscope = { version = "0.5.2", path = "../" } -pyroscope_pprofrs = {version = "0.2.1", path = "../pyroscope_backends/pyroscope_pprofrs" } -pyroscope_rbspy = {version = "0.2.1", path = "../pyroscope_backends/pyroscope_rbspy" } -pyroscope_pyspy = {version = "0.2.1", path = "../pyroscope_backends/pyroscope_pyspy" } +pyroscope_pprofrs = {version = "0.2.3", path = "../pyroscope_backends/pyroscope_pprofrs" } +pyroscope_rbspy = {version = "0.2.2", path = "../pyroscope_backends/pyroscope_rbspy" } +pyroscope_pyspy = {version = "0.2.3", path = "../pyroscope_backends/pyroscope_pyspy" } [dependencies.clap] -version = "=3.1.12" +version = "3.2" features = ["cargo", "derive"] [dev-dependencies] From 4dd7189c552bb84f62861acf4c9745e4250bb46e Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sun, 31 Jul 2022 12:41:48 -0500 Subject: [PATCH 39/70] imp(pyspy): harmonize API --- pyroscope_backends/pyroscope_pyspy/Cargo.toml | 2 +- pyroscope_backends/pyroscope_pyspy/src/lib.rs | 19 ++++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pyroscope_backends/pyroscope_pyspy/Cargo.toml b/pyroscope_backends/pyroscope_pyspy/Cargo.toml index 1c5db995..b2e58ab8 100644 --- a/pyroscope_backends/pyroscope_pyspy/Cargo.toml +++ b/pyroscope_backends/pyroscope_pyspy/Cargo.toml @@ -5,7 +5,7 @@ pyspy backend for Pyroscope Profiler. """ keywords = ["pyroscope", "profiler", "pyspy"] authors = ["Abid Omar "] -version = "0.2.3" +version = "0.2.4" edition = "2021" license = "Apache-2.0" homepage = "https://pyroscope.io" diff --git a/pyroscope_backends/pyroscope_pyspy/src/lib.rs b/pyroscope_backends/pyroscope_pyspy/src/lib.rs index 7d2174da..93b0b2a2 100644 --- a/pyroscope_backends/pyroscope_pyspy/src/lib.rs +++ b/pyroscope_backends/pyroscope_pyspy/src/lib.rs @@ -20,7 +20,7 @@ const LOG_TAG: &str = "Pyroscope::Pyspy"; /// Short-hand function for creating a new Pyspy backend. pub fn pyspy_backend(config: PyspyConfig) -> BackendImpl { // Clone BackendConfig to pass to the backend object. - let backend_config = config.backend_config.clone(); + let backend_config = config.backend_config; BackendImpl::new(Box::new(Pyspy::new(config)), Some(backend_config)) } @@ -41,7 +41,7 @@ pub struct PyspyConfig { /// Include subprocesses with_subprocesses: bool, /// Include idle time - include_idle: bool, + oncpu: bool, /// Detect Python GIL gil_only: bool, /// Profile native C extensions @@ -57,7 +57,7 @@ impl Default for PyspyConfig { lock_process: py_spy::config::LockingStrategy::NonBlocking, time_limit: None, with_subprocesses: false, - include_idle: false, + oncpu: false, gil_only: false, native: false, } @@ -146,11 +146,8 @@ impl PyspyConfig { } /// Include idle time - pub fn include_idle(self, include_idle: bool) -> Self { - PyspyConfig { - include_idle, - ..self - } + pub fn oncpu(self, oncpu: bool) -> Self { + PyspyConfig { oncpu, ..self } } /// Detect Python GIL @@ -217,7 +214,7 @@ impl Backend for Pyspy { Ok(self.config.sample_rate) } - fn set_config(&self, config: BackendConfig) {} + fn set_config(&self, _config: BackendConfig) {} fn get_config(&self) -> Result { Ok(self.config.backend_config) @@ -256,7 +253,7 @@ impl Backend for Pyspy { native: self.config.native, pid: self.config.pid, sampling_rate: self.config.sample_rate as u64, - include_idle: self.config.include_idle, + include_idle: self.config.oncpu, include_thread_ids: true, subprocesses: self.config.with_subprocesses, gil_only: self.config.gil_only, @@ -280,7 +277,7 @@ impl Backend for Pyspy { // create a new ruleset reference let ruleset = self.ruleset.clone(); - let backend_config = self.config.backend_config.clone(); + let backend_config = self.config.backend_config; self.sampler_thread = Some(std::thread::spawn(move || { // Get PID From 332fe8ea195b5560659634b5483e7f1cefa80340 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sun, 31 Jul 2022 13:50:34 -0500 Subject: [PATCH 40/70] imp: harmonize API and bump versions --- Cargo.toml | 2 +- .../pyroscope_pprofrs/Cargo.toml | 5 +++-- .../pyroscope_pprofrs/src/lib.rs | 11 ++++++---- pyroscope_backends/pyroscope_pyspy/Cargo.toml | 2 +- pyroscope_backends/pyroscope_rbspy/Cargo.toml | 4 ++-- pyroscope_backends/pyroscope_rbspy/src/lib.rs | 20 +++++++++++-------- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 780874f8..691874ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby appli """ keywords = ["pyroscope", "profiler", "profiling", "pprof"] authors = ["Abid Omar "] -version = "0.5.3" +version = "0.5.4" edition = "2021" license = "Apache-2.0" homepage = "https://pyroscope.io/docs/rust" diff --git a/pyroscope_backends/pyroscope_pprofrs/Cargo.toml b/pyroscope_backends/pyroscope_pprofrs/Cargo.toml index 668be761..1e6472e7 100644 --- a/pyroscope_backends/pyroscope_pprofrs/Cargo.toml +++ b/pyroscope_backends/pyroscope_pprofrs/Cargo.toml @@ -5,7 +5,7 @@ pprof-rs backend for Pyroscope Profiler. """ keywords = ["pyroscope", "profiler", "pprof-rs"] authors = ["Abid Omar "] -version = "0.2.3" +version = "0.2.4" edition = "2021" license = "Apache-2.0" homepage = "https://pyroscope.io" @@ -16,8 +16,9 @@ readme = "README.md" [dependencies] pprof = "0.10.0" -pyroscope = {version = "0.5.3", path = "../../" } +pyroscope = {version = "0.5.4", path = "../../" } thiserror ="1.0" +log = "0.4" [profile.dev] opt-level=0 diff --git a/pyroscope_backends/pyroscope_pprofrs/src/lib.rs b/pyroscope_backends/pyroscope_pprofrs/src/lib.rs index f74cc478..7ba632c1 100644 --- a/pyroscope_backends/pyroscope_pprofrs/src/lib.rs +++ b/pyroscope_backends/pyroscope_pprofrs/src/lib.rs @@ -13,8 +13,10 @@ use std::{ sync::{Arc, Mutex}, }; +const LOG_TAG: &str = "Pyroscope::Pprofrs"; + pub fn pprof_backend(config: PprofConfig) -> BackendImpl { - let backend_config = config.backend_config.clone(); + let backend_config = config.backend_config; BackendImpl::new(Box::new(Pprof::new(config)), Some(backend_config)) } @@ -101,11 +103,11 @@ impl std::fmt::Debug for Pprof<'_> { impl<'a> Pprof<'a> { /// Create a new Pprof Backend pub fn new(config: PprofConfig) -> Self { - let backend_config = config.backend_config.clone(); + let backend_config = config.backend_config; Pprof { buffer: Arc::new(Mutex::new(StackBuffer::default())), config, - backend_config: backend_config, + backend_config, inner_builder: Arc::new(Mutex::new(None)), guard: Arc::new(Mutex::new(None)), ruleset: Ruleset::default(), @@ -127,6 +129,7 @@ impl Backend for Pprof<'_> { } fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down sampler thread"); //drop(self.guard.take()); Ok(()) @@ -166,7 +169,7 @@ impl Backend for Pprof<'_> { Ok(reports) } - fn set_config(&self, config: BackendConfig) {} + fn set_config(&self, _config: BackendConfig) {} fn get_config(&self) -> Result { Ok(self.backend_config) diff --git a/pyroscope_backends/pyroscope_pyspy/Cargo.toml b/pyroscope_backends/pyroscope_pyspy/Cargo.toml index b2e58ab8..1abd9d2f 100644 --- a/pyroscope_backends/pyroscope_pyspy/Cargo.toml +++ b/pyroscope_backends/pyroscope_pyspy/Cargo.toml @@ -15,7 +15,7 @@ readme = "README.md" [dependencies] py-spy = "0.3.12" -pyroscope = { version = "0.5.3", path = "../../" } +pyroscope = { version = "0.5.4", path = "../../" } thiserror ="1.0" log = "0.4" diff --git a/pyroscope_backends/pyroscope_rbspy/Cargo.toml b/pyroscope_backends/pyroscope_rbspy/Cargo.toml index f7ac0c7a..bfee44c9 100644 --- a/pyroscope_backends/pyroscope_rbspy/Cargo.toml +++ b/pyroscope_backends/pyroscope_rbspy/Cargo.toml @@ -5,7 +5,7 @@ rbspy backend for Pyroscope Profiler. """ keywords = ["pyroscope", "profiler", "rbspy"] authors = ["Abid Omar "] -version = "0.2.2" +version = "0.2.3" edition = "2021" license = "Apache-2.0" homepage = "https://pyroscope.io" @@ -15,7 +15,7 @@ readme = "README.md" [dependencies] rbspy= {package = "rbspy-oncpu", version = "0.12.1"} -pyroscope = { version = "0.5.3", path = "../../" } +pyroscope = { version = "0.5.4", path = "../../" } thiserror ="1.0" log = "0.4" anyhow = "1.0.56" diff --git a/pyroscope_backends/pyroscope_rbspy/src/lib.rs b/pyroscope_backends/pyroscope_rbspy/src/lib.rs index 42b02307..372dcf46 100644 --- a/pyroscope_backends/pyroscope_rbspy/src/lib.rs +++ b/pyroscope_backends/pyroscope_rbspy/src/lib.rs @@ -15,10 +15,12 @@ use std::{ thread::JoinHandle, }; +const LOG_TAG: &str = "Pyroscope::Rbspy"; + /// Short-hand function for creating a new Rbspy backend. pub fn rbspy_backend(config: RbspyConfig) -> BackendImpl { // Clone BackendConfig to pass to the backend object. - let backend_config = config.backend_config.clone(); + let backend_config = config.backend_config; // Create a new backend object. BackendImpl::new(Box::new(Rbspy::new(config)), Some(backend_config)) @@ -40,7 +42,7 @@ pub struct RbspyConfig { /// Include subprocesses with_subprocesses: bool, /// Include Oncpu Time - on_cpu: bool, + oncpu: bool, } impl Default for RbspyConfig { @@ -52,7 +54,7 @@ impl Default for RbspyConfig { lock_process: false, time_limit: None, with_subprocesses: false, - on_cpu: false, + oncpu: false, } } } @@ -122,8 +124,8 @@ impl RbspyConfig { } /// Include idle time - pub fn on_cpu(self, on_cpu: bool) -> Self { - RbspyConfig { on_cpu, ..self } + pub fn oncpu(self, oncpu: bool) -> Self { + RbspyConfig { oncpu, ..self } } } @@ -184,7 +186,7 @@ impl Backend for Rbspy { Ok(self.config.sample_rate) } - fn set_config(&self, config: BackendConfig) {} + fn set_config(&self, _config: BackendConfig) {} fn get_config(&self) -> Result { Ok(self.config.backend_config) @@ -219,7 +221,7 @@ impl Backend for Rbspy { self.config.time_limit, self.config.with_subprocesses, None, - self.config.on_cpu, + self.config.oncpu, )); // Channel for Errors generated by the RubySpy Sampler @@ -259,7 +261,7 @@ impl Backend for Rbspy { // ruleset reference let ruleset = self.ruleset.clone(); - let backend_config = self.config.backend_config.clone(); + let backend_config = self.config.backend_config; let _: JoinHandle> = std::thread::spawn(move || { // Iterate over the StackTrace @@ -280,6 +282,8 @@ impl Backend for Rbspy { } fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down sampler thread"); + // Stop Sampler self.sampler .as_ref() From d4d093c64ba78dc12bb2d9d813cb9dfbbe203977 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sun, 31 Jul 2022 13:59:30 -0500 Subject: [PATCH 41/70] imp(cli): add auth_token/oncpu options --- pyroscope_backends/pyroscope_pyspy/src/lib.rs | 10 +++---- pyroscope_backends/pyroscope_rbspy/src/lib.rs | 10 +++---- pyroscope_cli/Cargo.lock | 9 +++--- pyroscope_cli/src/cli/lib.rs | 30 ++++++++++++++----- pyroscope_cli/src/core/profiler.rs | 11 ++++--- pyroscope_ffi/ruby/lib/pyroscope.rb | 6 ++-- 6 files changed, 47 insertions(+), 29 deletions(-) diff --git a/pyroscope_backends/pyroscope_pyspy/src/lib.rs b/pyroscope_backends/pyroscope_pyspy/src/lib.rs index 93b0b2a2..e466e8ca 100644 --- a/pyroscope_backends/pyroscope_pyspy/src/lib.rs +++ b/pyroscope_backends/pyroscope_pyspy/src/lib.rs @@ -39,7 +39,7 @@ pub struct PyspyConfig { /// Profiling duration (None for infinite) time_limit: Option, /// Include subprocesses - with_subprocesses: bool, + detect_subprocesses: bool, /// Include idle time oncpu: bool, /// Detect Python GIL @@ -56,7 +56,7 @@ impl Default for PyspyConfig { backend_config: BackendConfig::default(), lock_process: py_spy::config::LockingStrategy::NonBlocking, time_limit: None, - with_subprocesses: false, + detect_subprocesses: false, oncpu: false, gil_only: false, native: false, @@ -138,9 +138,9 @@ impl PyspyConfig { } /// Include subprocesses - pub fn with_subprocesses(self, with_subprocesses: bool) -> Self { + pub fn detect_subprocesses(self, detect_subprocesses: bool) -> Self { PyspyConfig { - with_subprocesses, + detect_subprocesses, ..self } } @@ -255,7 +255,7 @@ impl Backend for Pyspy { sampling_rate: self.config.sample_rate as u64, include_idle: self.config.oncpu, include_thread_ids: true, - subprocesses: self.config.with_subprocesses, + subprocesses: self.config.detect_subprocesses, gil_only: self.config.gil_only, duration, ..Config::default() diff --git a/pyroscope_backends/pyroscope_rbspy/src/lib.rs b/pyroscope_backends/pyroscope_rbspy/src/lib.rs index 372dcf46..831a50ed 100644 --- a/pyroscope_backends/pyroscope_rbspy/src/lib.rs +++ b/pyroscope_backends/pyroscope_rbspy/src/lib.rs @@ -40,7 +40,7 @@ pub struct RbspyConfig { /// Profiling duration. None for infinite. time_limit: Option, /// Include subprocesses - with_subprocesses: bool, + detect_subprocesses: bool, /// Include Oncpu Time oncpu: bool, } @@ -53,7 +53,7 @@ impl Default for RbspyConfig { backend_config: BackendConfig::default(), lock_process: false, time_limit: None, - with_subprocesses: false, + detect_subprocesses: false, oncpu: false, } } @@ -116,9 +116,9 @@ impl RbspyConfig { } /// Include subprocesses - pub fn with_subprocesses(self, with_subprocesses: bool) -> Self { + pub fn detect_subprocesses(self, detect_subprocesses: bool) -> Self { RbspyConfig { - with_subprocesses, + detect_subprocesses, ..self } } @@ -219,7 +219,7 @@ impl Backend for Rbspy { self.config.sample_rate, self.config.lock_process, self.config.time_limit, - self.config.with_subprocesses, + self.config.detect_subprocesses, None, self.config.oncpu, )); diff --git a/pyroscope_cli/Cargo.lock b/pyroscope_cli/Cargo.lock index a23af6f3..1945c1ac 100644 --- a/pyroscope_cli/Cargo.lock +++ b/pyroscope_cli/Cargo.lock @@ -1717,7 +1717,7 @@ dependencies = [ [[package]] name = "pyroscope" -version = "0.5.3" +version = "0.5.4" dependencies = [ "libc", "log", @@ -1760,8 +1760,9 @@ dependencies = [ [[package]] name = "pyroscope_pprofrs" -version = "0.2.3" +version = "0.2.4" dependencies = [ + "log", "pprof", "pyroscope", "thiserror", @@ -1769,7 +1770,7 @@ dependencies = [ [[package]] name = "pyroscope_pyspy" -version = "0.2.3" +version = "0.2.4" dependencies = [ "log", "py-spy", @@ -1779,7 +1780,7 @@ dependencies = [ [[package]] name = "pyroscope_rbspy" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "log", diff --git a/pyroscope_cli/src/cli/lib.rs b/pyroscope_cli/src/cli/lib.rs index 9c378628..b1c6ea6f 100644 --- a/pyroscope_cli/src/cli/lib.rs +++ b/pyroscope_cli/src/cli/lib.rs @@ -57,6 +57,13 @@ enum Commands { help = "application name used when uploading profiling data" )] application_name: Option, + #[clap( + name = "auth_token", + long = "auth-token", + value_name = "AUTH_TOKEN", + help = "Authentication token used when uploading profiling data" + )] + auth_token: Option, #[clap( name = "detect_subprocesses", long = "detect-subprocesses", @@ -91,10 +98,10 @@ enum Commands { )] blocking: bool, #[clap( - name = "pyspy_idle", - long = "pyspy-idle", - value_name = "PYSPY_IDLE", - help = "include idle threads for pyspy", + name = "oncpu", + long = "oncpu", + value_name = "ONCPU", + help = "enable oncpu mode. [supported by: rbspy, pyspy]", takes_value = false )] pyspy_idle: bool, @@ -168,6 +175,13 @@ enum Commands { help = "application name used when uploading profiling data" )] application_name: Option, + #[clap( + name = "auth_token", + long = "auth-token", + value_name = "AUTH_TOKEN", + help = "Authentication token used when uploading profiling data" + )] + auth_token: Option, #[clap( name = "detect_subprocesses", long = "detect-subprocesses", @@ -194,10 +208,10 @@ enum Commands { )] blocking: bool, #[clap( - name = "pyspy_idle", - long = "pyspy-idle", - value_name = "PYSPY_IDLE", - help = "include idle threads for pyspy", + name = "oncpu", + long = "oncpu", + value_name = "ONCPU", + help = "enable oncpu mode. [supported by: rbspy, pyspy]", takes_value = false )] pyspy_idle: bool, diff --git a/pyroscope_cli/src/core/profiler.rs b/pyroscope_cli/src/core/profiler.rs index 629cd5d0..dcf08f01 100644 --- a/pyroscope_cli/src/core/profiler.rs +++ b/pyroscope_cli/src/core/profiler.rs @@ -21,13 +21,15 @@ impl Profiler { let app_name: String = AppConfig::get::("application_name")?; + let auth_token: String = AppConfig::get::("auth_token")?; + let server_address: String = AppConfig::get::("server_address")?; let sample_rate: u32 = AppConfig::get::("sample_rate")?; let blocking: bool = AppConfig::get::("blocking")?; - let pyspy_idle: bool = AppConfig::get::("pyspy_idle")?; + let oncpu: bool = AppConfig::get::("oncpu")?; let pyspy_gil: bool = AppConfig::get::("pyspy_gil")?; let pyspy_native: bool = AppConfig::get::("pyspy_native")?; @@ -41,8 +43,8 @@ impl Profiler { let config = PyspyConfig::new(pid) .sample_rate(sample_rate) .lock_process(blocking) - .with_subprocesses(detect_subprocesses) - .include_idle(pyspy_idle) + .detect_subprocesses(detect_subprocesses) + .oncpu(oncpu) .gil_only(pyspy_gil) .native(pyspy_native); let backend = pyspy_backend(config); @@ -55,7 +57,8 @@ impl Profiler { let config = RbspyConfig::new(pid) .sample_rate(sample_rate) .lock_process(blocking) - .with_subprocesses(detect_subprocesses); + .oncpu(oncpu) + .detect_subprocesses(detect_subprocesses); let backend = rbspy_backend(config); PyroscopeAgent::builder(server_address, app_name) .backend(backend) diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index 076c5f78..747d3ca7 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -22,14 +22,14 @@ module Utils attach_function :thread_id, [], :uint64 end - Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :log_level, :sample_rate, :detect_subprocesses, :on_cpu, :report_pid, :report_thread_id, :tags) do + Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :log_level, :sample_rate, :detect_subprocesses, :oncpu, :report_pid, :report_thread_id, :tags) do def initialize(*) self.application_name = '' self.server_address = 'http://localhost:4040' self.auth_token = '' self.sample_rate = 100 self.detect_subprocesses = false - self.on_cpu = true + self.oncpu = true self.report_pid = false self.report_thread_id = false self.log_level = 'error' @@ -72,7 +72,7 @@ def configure @config.auth_token || "", @config.sample_rate || 100, @config.detect_subprocesses || false, - @config.on_cpu || false, + @config.oncpu || false, @config.report_pid || false, @config.report_thread_id || false, tags_to_string(@config.tags || {}) From de6057c022a03769c755683b611aa9c15cbd779d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 18:20:49 -0500 Subject: [PATCH 42/70] imp(cli): auth_token implementation --- pyroscope_cli/Cargo.lock | 41 ++++++++++--------- pyroscope_cli/Cargo.toml | 2 +- pyroscope_cli/src/core/profiler.rs | 31 ++++++++++---- .../src/resources/default_config.toml | 3 +- pyroscope_cli/src/utils/app_config.rs | 21 +++++++--- src/pyroscope.rs | 14 ++++++- 6 files changed, 76 insertions(+), 36 deletions(-) diff --git a/pyroscope_cli/Cargo.lock b/pyroscope_cli/Cargo.lock index 1945c1ac..b518968f 100644 --- a/pyroscope_cli/Cargo.lock +++ b/pyroscope_cli/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" +checksum = "c91f1f46651137be86f3a2b9a8359f9ab421d04d941c62b5982e1ca21113adf9" [[package]] name = "arc-swap" @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "3.2.0" +version = "3.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4fc4a94ae7c4e7c2e721893fce77ef8da529775e2d727d72de8753fcb2ad2a" +checksum = "ead064480dfc4880a10764488415a97fdd36a4cf1bb022d372f02e8faf8386e1" dependencies = [ "clap 3.2.16", ] @@ -342,7 +342,7 @@ dependencies = [ "lazy_static", "nom 5.1.2", "rust-ini", - "serde 1.0.140", + "serde 1.0.141", "serde-hjson", "serde_json", "toml", @@ -877,7 +877,7 @@ checksum = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36" dependencies = [ "backtrace", "os_type", - "serde 1.0.140", + "serde 1.0.141", "serde_derive", "termcolor", "toml", @@ -1707,7 +1707,7 @@ dependencies = [ "rand_distr", "regex", "remoteprocess", - "serde 1.0.140", + "serde 1.0.141", "serde_derive", "serde_json", "tempfile", @@ -1749,7 +1749,7 @@ dependencies = [ "pyroscope_pprofrs", "pyroscope_pyspy", "pyroscope_rbspy", - "serde 1.0.140", + "serde 1.0.141", "slog", "slog-async", "slog-scope", @@ -1905,7 +1905,7 @@ dependencies = [ "rand 0.8.5", "rbspy-ruby-structs", "remoteprocess", - "serde 1.0.140", + "serde 1.0.141", "serde_derive", "serde_json", "tempdir", @@ -2044,7 +2044,7 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-pemfile", - "serde 1.0.140", + "serde 1.0.141", "serde_json", "serde_urlencoded", "tokio", @@ -2242,9 +2242,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +checksum = "7af873f2c95b99fcb0bd0fe622a43e29514658873c8ceba88c4cb88833a22500" dependencies = [ "serde_derive", ] @@ -2263,9 +2263,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +checksum = "75743a150d003dd863b51dc809bcad0d73f2102c53632f1e954e738192a3413f" dependencies = [ "proc-macro2", "quote", @@ -2280,7 +2280,7 @@ checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ "itoa 1.0.2", "ryu", - "serde 1.0.140", + "serde 1.0.141", ] [[package]] @@ -2292,7 +2292,7 @@ dependencies = [ "form_urlencoded", "itoa 1.0.2", "ryu", - "serde 1.0.140", + "serde 1.0.141", ] [[package]] @@ -2370,7 +2370,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.11", + "time 0.3.12", ] [[package]] @@ -2616,11 +2616,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" +checksum = "74b7cc93fc23ba97fde84f7eea56c55d1ba183f495c6715defdfc7b9cb8c870f" dependencies = [ "itoa 1.0.2", + "js-sys", "libc", "num_threads", "time-macros", @@ -2706,7 +2707,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "serde 1.0.140", + "serde 1.0.141", ] [[package]] diff --git a/pyroscope_cli/Cargo.toml b/pyroscope_cli/Cargo.toml index 247c879f..d4216efb 100644 --- a/pyroscope_cli/Cargo.toml +++ b/pyroscope_cli/Cargo.toml @@ -33,7 +33,7 @@ slog-term = "2.8.0" slog-scope = "4.4.0" slog-async = "2.7.0" slog-stdlog = "4.1.0" -pyroscope = { version = "0.5.2", path = "../" } +pyroscope = { version = "0.5.4", path = "../" } pyroscope_pprofrs = {version = "0.2.3", path = "../pyroscope_backends/pyroscope_pprofrs" } pyroscope_rbspy = {version = "0.2.2", path = "../pyroscope_backends/pyroscope_rbspy" } pyroscope_pyspy = {version = "0.2.3", path = "../pyroscope_backends/pyroscope_pyspy" } diff --git a/pyroscope_cli/src/core/profiler.rs b/pyroscope_cli/src/core/profiler.rs index dcf08f01..3686b453 100644 --- a/pyroscope_cli/src/core/profiler.rs +++ b/pyroscope_cli/src/core/profiler.rs @@ -47,11 +47,19 @@ impl Profiler { .oncpu(oncpu) .gil_only(pyspy_gil) .native(pyspy_native); + let backend = pyspy_backend(config); - PyroscopeAgent::builder(server_address, app_name) - .backend(backend) - .tags(tags) - .build()? + + let mut builder = PyroscopeAgent::default_builder(); + builder = builder.url(server_address); + builder = builder.application_name(app_name); + + // There must be a better way to do this, hopefully as clap supports Option + if auth_token.len() > 0 { + builder = builder.auth_token(auth_token); + } + + builder.backend(backend).tags(tags).build()? } Spy::Rbspy => { let config = RbspyConfig::new(pid) @@ -60,10 +68,17 @@ impl Profiler { .oncpu(oncpu) .detect_subprocesses(detect_subprocesses); let backend = rbspy_backend(config); - PyroscopeAgent::builder(server_address, app_name) - .backend(backend) - .tags(tags) - .build()? + + let mut builder = PyroscopeAgent::default_builder(); + builder = builder.url(server_address); + builder = builder.application_name(app_name); + + // There must be a better way to do this, hopefully as clap supports Option + if auth_token.len() > 0 { + builder = builder.auth_token(auth_token); + } + + builder.backend(backend).tags(tags).build()? } }; diff --git a/pyroscope_cli/src/resources/default_config.toml b/pyroscope_cli/src/resources/default_config.toml index 7fa7927f..d2246b3e 100644 --- a/pyroscope_cli/src/resources/default_config.toml +++ b/pyroscope_cli/src/resources/default_config.toml @@ -2,9 +2,10 @@ detect_subprocesses = false log_level = "info" no_logging = false blocking = false -pyspy_idle = false +oncpu = false pyspy_gil = false pyspy_native = false sample_rate = 100 server_address = "http://localhost:4040" +auth_token = "" debug = false diff --git a/pyroscope_cli/src/utils/app_config.rs b/pyroscope_cli/src/utils/app_config.rs index 24e806a8..ffb0a53a 100644 --- a/pyroscope_cli/src/utils/app_config.rs +++ b/pyroscope_cli/src/utils/app_config.rs @@ -19,12 +19,13 @@ pub struct AppConfig { pub pid: Option, pub spy_name: Spy, pub application_name: Option, + pub auth_token: Option, pub detect_subprocesses: Option, pub no_logging: Option, pub log_level: LogLevel, pub no_root_drop: Option, pub blocking: Option, - pub pyspy_idle: Option, + pub onpcu: Option, pub pyspy_gil: Option, pub pyspy_native: Option, pub sample_rate: Option, @@ -77,8 +78,8 @@ impl AppConfig { if sub_connect.is_present("blocking") { AppConfig::set("blocking", "true")?; } - if sub_connect.is_present("pyspy_idle") { - AppConfig::set("pyspy_idle", "true")?; + if sub_connect.is_present("oncpu") { + AppConfig::set("oncpu", "true")?; } if sub_connect.is_present("pyspy_gil") { AppConfig::set("pyspy_gil", "true")?; @@ -109,6 +110,11 @@ impl AppConfig { AppConfig::set("application_name", application_name)?; } } + if sub_connect.is_present("auth_token") { + if let Some(auth_token) = sub_connect.value_of("auth_token") { + AppConfig::set("auth_token", auth_token)?; + } + } if sub_connect.is_present("detect_subprocesses") { AppConfig::set("detect_subprocesses", "true")?; } @@ -157,8 +163,8 @@ impl AppConfig { if sub_exec.is_present("blocking") { AppConfig::set("blocking", "true")?; } - if sub_exec.is_present("pyspy_idle") { - AppConfig::set("pyspy_idle", "true")?; + if sub_exec.is_present("oncpu") { + AppConfig::set("oncpu", "true")?; } if sub_exec.is_present("pyspy_gil") { AppConfig::set("pyspy_gil", "true")?; @@ -189,6 +195,11 @@ impl AppConfig { AppConfig::set("application_name", application_name)?; } } + if sub_exec.is_present("auth_token") { + if let Some(auth_token) = sub_exec.value_of("auth_token") { + AppConfig::set("auth_token", auth_token)?; + } + } if sub_exec.is_present("detect_subprocesses") { AppConfig::set("detect_subprocesses", "true")?; } diff --git a/src/pyroscope.rs b/src/pyroscope.rs index 08127a42..32edae81 100644 --- a/src/pyroscope.rs +++ b/src/pyroscope.rs @@ -454,7 +454,7 @@ impl PyroscopeAgent { } impl PyroscopeAgent { - /// Short-hand for PyroscopeAgentBuilder::build(). This is a convenience method. + /// Short-hand for PyroscopeAgentBuilder::new(url, application_name). This is a convenience method. /// /// # Example /// ```ignore @@ -464,6 +464,18 @@ impl PyroscopeAgent { // Build PyroscopeAgent PyroscopeAgentBuilder::new(url, application_name) } + + /// Short-hand for PyroscopeAgentBuilder::default(). This is a convenience method. + /// Default URL is "http://localhost:4040". Default application name is randomly generated. + /// + /// # Example + /// ```ignore + /// let agent = PyroscopeAgent::default_builder().build()?; + /// ``` + pub fn default_builder() -> PyroscopeAgentBuilder { + // Build PyroscopeAgent + PyroscopeAgentBuilder::default() + } } impl PyroscopeAgent { From 34386aaafb8bf0e684615d2213ccd5c321635540 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 18:53:30 -0500 Subject: [PATCH 43/70] wip: single test file --- .github/workflows/ci-ffi-python.yml | 46 +++-------------- pyroscope_ffi/python/scripts/tests/test.py | 59 ++++++++++++++++++++++ 2 files changed, 67 insertions(+), 38 deletions(-) create mode 100755 pyroscope_ffi/python/scripts/tests/test.py diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 354b372a..ea4295c9 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -47,6 +47,13 @@ jobs: path: pyroscope_ffi/python/dist/* linux-test: + strategy: + fail-fast: false + matrix: + PYROSCOPE_DETECT_SUBPROCESSES: [True, False] + PYROSCOPE_ONCPU: [True, False] + PYROSCOPE_GIL_ONLY: [True, False] + PYROSCOPE_NATIVE: [True, False] needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest @@ -62,48 +69,11 @@ jobs: with: submodules: recursive - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/all.py + run: pyroscope_ffi/python/scripts/tests/test.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - linux-arm-build: strategy: diff --git a/pyroscope_ffi/python/scripts/tests/test.py b/pyroscope_ffi/python/scripts/tests/test.py new file mode 100755 index 00000000..a2a0c241 --- /dev/null +++ b/pyroscope_ffi/python/scripts/tests/test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import hashlib +import os +import threading +import logging + +import pyroscope + + +# Set python logging level to DEBUG +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# Configure Pyroscope +pyroscope.configure( + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-all', + server_address = "https://ingest.pyroscope.cloud", + auth_token = os.getenv("PYROSCOPE_API_TOKEN"), + enable_logging =True, + detect_subprocesses = os.getenv("PYROSCOPE_DETECT_SUBPROCESSES"), + oncpu = os.getenv("PYROSCOPE_ONCPU"), + gil_only = os.getenv("PYROSCOPE_GIL_ONLY"), + report_pid = True, + report_thread_id = True, + report_thread_name = True, + tags = { + "detect_subprocesses": os.getenv("PYROSCOPE_DETECT_SUBPROCESSES"), + "oncpu": os.getenv("PYROSCOPE_ONCPU"), + "gil_only": os.getenv("PYROSCOPE_GIL_ONLY"), + } +) + + +def hash(string): + string = string.encode() + string = hashlib.sha256(string).hexdigest() + + return string + +def multihash(string): + for i in range(0, 25510055): + string = hash(string) + return string + +def multihash2(string): + for i in range(0, 25510055): + string = hash(string) + return string + +thread_1 = threading.Thread(target=multihash, args=('abc',)) +thread_2 = threading.Thread(target=multihash2, args=('abc',)) + +thread_1.start() +thread_2.start() + +thread_1.join() +thread_2.join() + +pyroscope.shutdown() From 0949e405da6aa5782eb19064c5b461734c3c3131 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 19:01:55 -0500 Subject: [PATCH 44/70] wip: update ffi --- pyroscope_ffi/python/lib/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyroscope_ffi/python/lib/src/lib.rs b/pyroscope_ffi/python/lib/src/lib.rs index 405c0c54..478e07a9 100644 --- a/pyroscope_ffi/python/lib/src/lib.rs +++ b/pyroscope_ffi/python/lib/src/lib.rs @@ -75,8 +75,8 @@ pub extern "C" fn initialize_agent( let mut pyspy_config = PyspyConfig::new(pid.try_into().unwrap()) .sample_rate(sample_rate) .lock_process(false) - .with_subprocesses(detect_subprocesses) - .include_idle(!oncpu) + .detect_subprocesses(detect_subprocesses) + .oncpu(oncpu) .native(native) .gil_only(gil_only); From abf824ffd1101f9463511770850cc7f8c4979f8b Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 21:53:58 -0500 Subject: [PATCH 45/70] wip: add env variables --- .github/workflows/ci-ffi-python.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index ea4295c9..0ba57b2a 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -74,7 +74,10 @@ jobs: PYROSCOPE_RUN_ID: ${{ github.run_id }} PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - + PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} + PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} linux-arm-build: strategy: fail-fast: false From 7b748ec467663c56b8312d1a2240a5a37268df96 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 22:06:30 -0500 Subject: [PATCH 46/70] wip: use an integer --- .github/workflows/ci-ffi-python.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 0ba57b2a..1e2fdc1d 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -50,10 +50,10 @@ jobs: strategy: fail-fast: false matrix: - PYROSCOPE_DETECT_SUBPROCESSES: [True, False] - PYROSCOPE_ONCPU: [True, False] - PYROSCOPE_GIL_ONLY: [True, False] - PYROSCOPE_NATIVE: [True, False] + PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + PYROSCOPE_ONCPU: [1, 0] + PYROSCOPE_GIL_ONLY: [1, 0] + PYROSCOPE_NATIVE: [1, 0] needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest From 82be72d5afae4b734199a8d9d1b9823fbe2117ef Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 22:21:16 -0500 Subject: [PATCH 47/70] wip: python bool --- pyroscope_ffi/python/scripts/tests/test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyroscope_ffi/python/scripts/tests/test.py b/pyroscope_ffi/python/scripts/tests/test.py index a2a0c241..ac9c7cc0 100755 --- a/pyroscope_ffi/python/scripts/tests/test.py +++ b/pyroscope_ffi/python/scripts/tests/test.py @@ -15,18 +15,18 @@ pyroscope.configure( application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-all', server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), + auth_token = f'{os.getenv("PYROSCOPE_API_TOKEN")}', enable_logging =True, - detect_subprocesses = os.getenv("PYROSCOPE_DETECT_SUBPROCESSES"), - oncpu = os.getenv("PYROSCOPE_ONCPU"), - gil_only = os.getenv("PYROSCOPE_GIL_ONLY"), + detect_subprocesses = os.getenv("PYROSCOPE_DETECT_SUBPROCESSES") == "true", + oncpu = os.getenv("PYROSCOPE_ONCPU") == "true", + gil_only = os.getenv("PYROSCOPE_GIL_ONLY") == "true", report_pid = True, report_thread_id = True, report_thread_name = True, tags = { - "detect_subprocesses": os.getenv("PYROSCOPE_DETECT_SUBPROCESSES"), - "oncpu": os.getenv("PYROSCOPE_ONCPU"), - "gil_only": os.getenv("PYROSCOPE_GIL_ONLY"), + "detect_subprocesses": f'{os.getenv("PYROSCOPE_DETECT_SUBPROCESSES")}', + "oncpu": f'{os.getenv("PYROSCOPE_ONCPU")}', + "gil_only": f'{os.getenv("PYROSCOPE_GIL_ONLY")}', } ) From ffb8b813bcc92969ba7dcb297fa61f96430fd0d5 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 1 Aug 2022 22:37:06 -0500 Subject: [PATCH 48/70] wip: python version matrix --- .github/workflows/ci-ffi-python.yml | 7 +++++++ pyroscope_ffi/python/scripts/tests/test.py | 1 + 2 files changed, 8 insertions(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 1e2fdc1d..43da320e 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -54,10 +54,16 @@ jobs: PYROSCOPE_ONCPU: [1, 0] PYROSCOPE_GIL_ONLY: [1, 0] PYROSCOPE_NATIVE: [1, 0] + PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.PYTHON_VERSION }} + architecture: x64 - uses: actions/download-artifact@master with: name: "linux.whl" @@ -78,6 +84,7 @@ jobs: PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} + PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} linux-arm-build: strategy: fail-fast: false diff --git a/pyroscope_ffi/python/scripts/tests/test.py b/pyroscope_ffi/python/scripts/tests/test.py index ac9c7cc0..5c7870fc 100755 --- a/pyroscope_ffi/python/scripts/tests/test.py +++ b/pyroscope_ffi/python/scripts/tests/test.py @@ -27,6 +27,7 @@ "detect_subprocesses": f'{os.getenv("PYROSCOPE_DETECT_SUBPROCESSES")}', "oncpu": f'{os.getenv("PYROSCOPE_ONCPU")}', "gil_only": f'{os.getenv("PYROSCOPE_GIL_ONLY")}', + "version": f'{os.getenv("PYTHON_VERSION")}', } ) From 8c55bb5546a562c170061734ce25abc018feb173 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 10:22:18 +0300 Subject: [PATCH 49/70] wip: try pip3 --- .github/workflows/ci-ffi-python.yml | 196 ++++++++++------------------ 1 file changed, 68 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 43da320e..8586e6d5 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -46,132 +46,73 @@ jobs: name: "linux.whl" path: pyroscope_ffi/python/dist/* - linux-test: - strategy: - fail-fast: false - matrix: - PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] - PYROSCOPE_ONCPU: [1, 0] - PYROSCOPE_GIL_ONLY: [1, 0] - PYROSCOPE_NATIVE: [1, 0] - PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] - needs: ['linux-build'] - name: Linux Test - runs-on: ubuntu-latest - steps: - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.PYTHON_VERSION }} - architecture: x64 - - uses: actions/download-artifact@master - with: - name: "linux.whl" - path: "${{github.workspace}}/python" - - - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "cd ${{ github.workspace }}/python && pip install *.whl" - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/test.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} - PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} - PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} - PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} - PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - linux-arm-build: - strategy: - fail-fast: false - matrix: - build-arch: - - manylinux2014_aarch64 - - name: Linux - ${{ matrix.build-arch }} - runs-on: [self-hosted, Linux, ARM64] - - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Build in Docker - run: pyroscope_ffi/python/scripts/docker.sh - env: - BUILD_ARCH: ${{ matrix.build-arch }} - - - uses: actions/upload-artifact@v2 - with: - name: "linux-arm.whl" - path: pyroscope_ffi/python/dist/* - - linux-arm-test: - needs: ['linux-arm-build'] - name: Linux ARM Test - runs-on: [self-hosted, Linux, ARM64] - steps: - - uses: actions/download-artifact@master - with: - name: "linux-arm.whl" - path: "${{github.workspace}}/python" - - - run: "apt-get update && apt-get install python3 python3-pip" - - - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "cd ${{ github.workspace }}/python && pip install *.whl" - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/all.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: aarch64-linux - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - macos-build: + #linux-test: + #strategy: + #fail-fast: false + #matrix: + #PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + #PYROSCOPE_ONCPU: [1, 0] + #PYROSCOPE_GIL_ONLY: [1, 0] + #PYROSCOPE_NATIVE: [1, 0] + #PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] + #needs: ['linux-build'] + #name: Linux Test + #runs-on: ubuntu-latest + #steps: + #- name: Set up Python + #uses: actions/setup-python@v4 + #with: + #python-version: ${{ matrix.PYTHON_VERSION }} + #architecture: x64 + #- uses: actions/download-artifact@master + #with: + #name: "linux.whl" + #path: "${{github.workspace}}/python" + + #- run: "cd ${{ github.workspace }}/python && ls -l" + #- run: "cd ${{ github.workspace }}/python && pip install *.whl" + #- uses: actions/checkout@v2 + #with: + #submodules: recursive + #- name: Run Python Script + #run: pyroscope_ffi/python/scripts/tests/test.py + #env: + #PYROSCOPE_RUN_ID: ${{ github.run_id }} + #PYROSCOPE_ARCH: x86-64-linux + #PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + #PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + #PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + #PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} + #PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} + #PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} + + #linux-arm-build: + #strategy: + #fail-fast: false + #matrix: + #build-arch: + #- manylinux2014_aarch64 + + #name: Linux - ${{ matrix.build-arch }} + #runs-on: [self-hosted, Linux, ARM64] + + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 + #with: + #submodules: recursive + + #- name: Build in Docker + #run: pyroscope_ffi/python/scripts/docker.sh + #env: + #BUILD_ARCH: ${{ matrix.build-arch }} + + #- uses: actions/upload-artifact@v2 + #with: + #name: "linux-arm.whl" + #path: pyroscope_ffi/python/dist/* + + macos-build: strategy: fail-fast: false matrix: @@ -227,9 +168,8 @@ jobs: name: x86_64-apple-darwin path: "${{github.workspace}}/python" - - run: "pip install --upgrade pip" - run: "cd ${{ github.workspace }}/python && ls -l" - - run: "cd ${{ github.workspace }}/python && pip install *.whl" + - run: "cd ${{ github.workspace }}/python && pip3 install *.whl" - uses: actions/checkout@v2 with: submodules: recursive From a9e2ee4697ad4e7abc16611e948b42aac5528fd2 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 10:23:57 +0300 Subject: [PATCH 50/70] wip: yaml formatting --- .github/workflows/ci-ffi-python.yml | 52 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 8586e6d5..f90313e6 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -15,36 +15,36 @@ on: - '**' jobs: - linux-build: - strategy: - fail-fast: false - matrix: - build-arch: - - manylinux2010_x86_64 + #linux-build: + #strategy: + #fail-fast: false + #matrix: + #build-arch: + #- manylinux2010_x86_64 - name: Linux - ${{ matrix.build-arch }} - runs-on: ubuntu-latest + #name: Linux - ${{ matrix.build-arch }} + #runs-on: ubuntu-latest - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v2 - with: - submodules: recursive + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 + #with: + #submodules: recursive - - if: matrix.build-arch == 'manylinux2014_aarch64' - uses: docker/setup-qemu-action@v1 - with: - platforms: arm64 + #- if: matrix.build-arch == 'manylinux2014_aarch64' + #uses: docker/setup-qemu-action@v1 + #with: + #platforms: arm64 - - name: Build in Docker - run: pyroscope_ffi/python/scripts/docker.sh - env: - BUILD_ARCH: ${{ matrix.build-arch }} + #- name: Build in Docker + #run: pyroscope_ffi/python/scripts/docker.sh + #env: + #BUILD_ARCH: ${{ matrix.build-arch }} - - uses: actions/upload-artifact@v2 - with: - name: "linux.whl" - path: pyroscope_ffi/python/dist/* + #- uses: actions/upload-artifact@v2 + #with: + #name: "linux.whl" + #path: pyroscope_ffi/python/dist/* #linux-test: #strategy: @@ -112,7 +112,7 @@ jobs: #name: "linux-arm.whl" #path: pyroscope_ffi/python/dist/* - macos-build: + macos-build: strategy: fail-fast: false matrix: From 2971d99a99a03a69f4d64cb204cd244e9da944c6 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 10:43:24 +0300 Subject: [PATCH 51/70] wip: use matrix for macos tests --- .github/workflows/ci-ffi-python.yml | 60 +++++++++-------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index f90313e6..cbdb69db 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -141,6 +141,7 @@ jobs: with: python-version: 3.9 + - name: Build Wheel run: | pip install wheel @@ -155,6 +156,14 @@ jobs: path: pyroscope_ffi/python/dist/* macos-test: + strategy: + fail-fast: false + matrix: + PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + PYROSCOPE_ONCPU: [1, 0] + PYROSCOPE_GIL_ONLY: [1, 0] + PYROSCOPE_NATIVE: [1, 0] + PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] needs: ['macos-build'] name: Macos Test runs-on: macos-11.0 @@ -162,7 +171,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.x + python-version: ${{ matrix.PYTHON_VERSION }} + architecture: x64 - uses: actions/download-artifact@master with: name: x86_64-apple-darwin @@ -174,49 +184,17 @@ jobs: with: submodules: recursive - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/all.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu-gil.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-gil.py + run: pyroscope_ffi/python/scripts/tests/test.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos + PYROSCOPE_ARCH: x86-64-linux PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub-oncpu.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - name: Run Python Script - run: pyroscope_ffi/python/scripts/tests/sub.py - env: - PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-macos - PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - - + PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} + PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} + PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} + sdist-build: name: sdist runs-on: ubuntu-latest From 22125f8ee7ef627394d63645242014c37e72658a Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 11:35:42 +0300 Subject: [PATCH 52/70] wip: trying something --- .github/workflows/ci-ffi-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index cbdb69db..13d43812 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -145,7 +145,7 @@ jobs: - name: Build Wheel run: | pip install wheel - python setup.py bdist_wheel -p ${{ matrix.py-platform }} + python3 setup.py bdist_wheel -p ${{ matrix.py-platform }} working-directory: pyroscope_ffi/python env: CARGO_BUILD_TARGET: ${{ matrix.target }} @@ -187,7 +187,7 @@ jobs: run: pyroscope_ffi/python/scripts/tests/test.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-linux + PYROSCOPE_ARCH: x86-64-darwin PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} From 1cd8974caac98f3694864c5c1d1ef3ca0b0b01b9 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 11:38:38 +0300 Subject: [PATCH 53/70] wip: get python version details --- .github/workflows/ci-ffi-python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 13d43812..5e56b7e7 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -178,6 +178,8 @@ jobs: name: x86_64-apple-darwin path: "${{github.workspace}}/python" + - run: "python --version" + - run: "python3 --version" - run: "cd ${{ github.workspace }}/python && ls -l" - run: "cd ${{ github.workspace }}/python && pip3 install *.whl" - uses: actions/checkout@v2 From 242ed925f74b2ab69aa0d26ed79b701160d8fdfd Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 12:19:52 +0300 Subject: [PATCH 54/70] wip: change macos version --- .github/workflows/ci-ffi-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 5e56b7e7..b7f8cf55 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -166,7 +166,7 @@ jobs: PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] needs: ['macos-build'] name: Macos Test - runs-on: macos-11.0 + runs-on: macos-12.0 steps: - name: Set up Python uses: actions/setup-python@v4 From a126005a4ae74e0bf41b9a036d951c8d8a6fe23d Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 12:45:03 +0300 Subject: [PATCH 55/70] wip: remove test scripts --- .github/workflows/ci-ffi-python.yml | 194 +++++++++--------- pyroscope_ffi/python/scripts/tests/all.py | 54 ----- pyroscope_ffi/python/scripts/tests/gil.py | 54 ----- .../python/scripts/tests/oncpu-gil.py | 54 ----- pyroscope_ffi/python/scripts/tests/oncpu.py | 54 ----- pyroscope_ffi/python/scripts/tests/sub-gil.py | 54 ----- .../python/scripts/tests/sub-oncpu.py | 54 ----- pyroscope_ffi/python/scripts/tests/sub.py | 54 ----- 8 files changed, 97 insertions(+), 475 deletions(-) delete mode 100755 pyroscope_ffi/python/scripts/tests/all.py delete mode 100755 pyroscope_ffi/python/scripts/tests/gil.py delete mode 100755 pyroscope_ffi/python/scripts/tests/oncpu-gil.py delete mode 100755 pyroscope_ffi/python/scripts/tests/oncpu.py delete mode 100755 pyroscope_ffi/python/scripts/tests/sub-gil.py delete mode 100755 pyroscope_ffi/python/scripts/tests/sub-oncpu.py delete mode 100755 pyroscope_ffi/python/scripts/tests/sub.py diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index b7f8cf55..13d29e6e 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -15,102 +15,102 @@ on: - '**' jobs: - #linux-build: - #strategy: - #fail-fast: false - #matrix: - #build-arch: - #- manylinux2010_x86_64 - - #name: Linux - ${{ matrix.build-arch }} - #runs-on: ubuntu-latest - - #steps: - #- uses: AutoModality/action-clean@v1 - #- uses: actions/checkout@v2 - #with: - #submodules: recursive - - #- if: matrix.build-arch == 'manylinux2014_aarch64' - #uses: docker/setup-qemu-action@v1 - #with: - #platforms: arm64 - - #- name: Build in Docker - #run: pyroscope_ffi/python/scripts/docker.sh - #env: - #BUILD_ARCH: ${{ matrix.build-arch }} - - #- uses: actions/upload-artifact@v2 - #with: - #name: "linux.whl" - #path: pyroscope_ffi/python/dist/* - - #linux-test: - #strategy: - #fail-fast: false - #matrix: - #PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] - #PYROSCOPE_ONCPU: [1, 0] - #PYROSCOPE_GIL_ONLY: [1, 0] - #PYROSCOPE_NATIVE: [1, 0] - #PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] - #needs: ['linux-build'] - #name: Linux Test - #runs-on: ubuntu-latest - #steps: - #- name: Set up Python - #uses: actions/setup-python@v4 - #with: - #python-version: ${{ matrix.PYTHON_VERSION }} - #architecture: x64 - #- uses: actions/download-artifact@master - #with: - #name: "linux.whl" - #path: "${{github.workspace}}/python" - - #- run: "cd ${{ github.workspace }}/python && ls -l" - #- run: "cd ${{ github.workspace }}/python && pip install *.whl" - #- uses: actions/checkout@v2 - #with: - #submodules: recursive - #- name: Run Python Script - #run: pyroscope_ffi/python/scripts/tests/test.py - #env: - #PYROSCOPE_RUN_ID: ${{ github.run_id }} - #PYROSCOPE_ARCH: x86-64-linux - #PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} - #PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} - #PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} - #PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} - #PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} - #PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} - - #linux-arm-build: - #strategy: - #fail-fast: false - #matrix: - #build-arch: - #- manylinux2014_aarch64 - - #name: Linux - ${{ matrix.build-arch }} - #runs-on: [self-hosted, Linux, ARM64] - - #steps: - #- uses: AutoModality/action-clean@v1 - #- uses: actions/checkout@v2 - #with: - #submodules: recursive - - #- name: Build in Docker - #run: pyroscope_ffi/python/scripts/docker.sh - #env: - #BUILD_ARCH: ${{ matrix.build-arch }} - - #- uses: actions/upload-artifact@v2 - #with: - #name: "linux-arm.whl" - #path: pyroscope_ffi/python/dist/* + linux-build: + strategy: + fail-fast: false + matrix: + build-arch: + - manylinux2010_x86_64 + + name: Linux - ${{ matrix.build-arch }} + runs-on: ubuntu-latest + + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 + with: + submodules: recursive + + - if: matrix.build-arch == 'manylinux2014_aarch64' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + + - name: Build in Docker + run: pyroscope_ffi/python/scripts/docker.sh + env: + BUILD_ARCH: ${{ matrix.build-arch }} + + - uses: actions/upload-artifact@v2 + with: + name: "linux.whl" + path: pyroscope_ffi/python/dist/* + + linux-test: + strategy: + fail-fast: false + matrix: + PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + PYROSCOPE_ONCPU: [1, 0] + PYROSCOPE_GIL_ONLY: [1, 0] + PYROSCOPE_NATIVE: [1, 0] + PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] + needs: ['linux-build'] + name: Linux Test + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.PYTHON_VERSION }} + architecture: x64 + - uses: actions/download-artifact@master + with: + name: "linux.whl" + path: "${{github.workspace}}/python" + + - run: "cd ${{ github.workspace }}/python && ls -l" + - run: "cd ${{ github.workspace }}/python && pip install *.whl" + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Run Python Script + run: pyroscope_ffi/python/scripts/tests/test.py + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + PYROSCOPE_GIL_ONLY: ${{ matrix.PYROSCOPE_GIL_ONLY }} + PYROSCOPE_NATIVE: ${{ matrix.PYROSCOPE_NATIVE }} + PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} + + linux-arm-build: + strategy: + fail-fast: false + matrix: + build-arch: + - manylinux2014_aarch64 + + name: Linux - ${{ matrix.build-arch }} + runs-on: [self-hosted, Linux, ARM64] + + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Build in Docker + run: pyroscope_ffi/python/scripts/docker.sh + env: + BUILD_ARCH: ${{ matrix.build-arch }} + + - uses: actions/upload-artifact@v2 + with: + name: "linux-arm.whl" + path: pyroscope_ffi/python/dist/* macos-build: strategy: @@ -155,7 +155,7 @@ jobs: name: ${{ matrix.target }} path: pyroscope_ffi/python/dist/* - macos-test: + macos-intel-test: strategy: fail-fast: false matrix: diff --git a/pyroscope_ffi/python/scripts/tests/all.py b/pyroscope_ffi/python/scripts/tests/all.py deleted file mode 100755 index 46197d85..00000000 --- a/pyroscope_ffi/python/scripts/tests/all.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-all', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging =True, - detect_subprocesses=True, - oncpu=True, - gil_only=True, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/gil.py b/pyroscope_ffi/python/scripts/tests/gil.py deleted file mode 100755 index 4e6049ef..00000000 --- a/pyroscope_ffi/python/scripts/tests/gil.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-gil', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=False, - oncpu=False, - gil_only=True, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py b/pyroscope_ffi/python/scripts/tests/oncpu-gil.py deleted file mode 100755 index eb510af1..00000000 --- a/pyroscope_ffi/python/scripts/tests/oncpu-gil.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-oncpu:gil', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=False, - oncpu=True, - gil_only=True, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/oncpu.py b/pyroscope_ffi/python/scripts/tests/oncpu.py deleted file mode 100755 index 33945ca6..00000000 --- a/pyroscope_ffi/python/scripts/tests/oncpu.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-onpcu', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=False, - oncpu=True, - gil_only=False, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub-gil.py b/pyroscope_ffi/python/scripts/tests/sub-gil.py deleted file mode 100755 index fd99fc83..00000000 --- a/pyroscope_ffi/python/scripts/tests/sub-gil.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name =f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub:gil', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=True, - oncpu=False, - gil_only=True, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py b/pyroscope_ffi/python/scripts/tests/sub-oncpu.py deleted file mode 100755 index 91c0d3b1..00000000 --- a/pyroscope_ffi/python/scripts/tests/sub-oncpu.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub:oncpu', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=True, - oncpu=True, - gil_only=False, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() diff --git a/pyroscope_ffi/python/scripts/tests/sub.py b/pyroscope_ffi/python/scripts/tests/sub.py deleted file mode 100755 index 1b7ee40b..00000000 --- a/pyroscope_ffi/python/scripts/tests/sub.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import os -import threading -import logging - -import pyroscope - - -# Set python logging level to DEBUG -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -# Configure Pyroscope -pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-sub', - server_address = "https://ingest.pyroscope.cloud", - auth_token = os.getenv("PYROSCOPE_API_TOKEN"), - enable_logging=True, - detect_subprocesses=True, - oncpu=False, - gil_only=False, - report_pid=True, - report_thread_id=True, - report_thread_name=True, -) - - -def hash(string): - string = string.encode() - string = hashlib.sha256(string).hexdigest() - - return string - -def multihash(string): - for i in range(0, 25510055): - string = hash(string) - return string - -def multihash2(string): - for i in range(0, 25510055): - string = hash(string) - return string - -thread_1 = threading.Thread(target=multihash, args=('abc',)) -thread_2 = threading.Thread(target=multihash2, args=('abc',)) - -thread_1.start() -thread_2.start() - -thread_1.join() -thread_2.join() - -pyroscope.shutdown() From 87569a3ffbab1c56517f072434f045c10bc2c8bd Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 13:03:00 +0300 Subject: [PATCH 56/70] wip: ruby test file --- pyroscope_ffi/ruby/scripts/tests/test.rb | 58 ++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 pyroscope_ffi/ruby/scripts/tests/test.rb diff --git a/pyroscope_ffi/ruby/scripts/tests/test.rb b/pyroscope_ffi/ruby/scripts/tests/test.rb new file mode 100755 index 00000000..1498af3f --- /dev/null +++ b/pyroscope_ffi/ruby/scripts/tests/test.rb @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby + +require "pyroscope" +require "pyroscope/version" + +puts Pyroscope::VERSION +puts RUBY_VERSION + +Pyroscope.configure do |config| + config.application_name = "new.fork.ruby" + config.server_address = "https://ingest.pyroscope.cloud" + config.auth_token = ENV["PYROSCOPE_API_TOKEN"] + config.detect_subprocesses = ENV["PYROSCOPE_DETECT_SUBPROCESSES"] + config.oncpu = ENV["PYROSCOPE_ONCPU"] + config.log_level = "trace" + config.report_pid = true + config.report_thread_id = true + config.tags = { + :region => "us-east", + :detect_subprocesses => ENV["PYROSCOPE_DETECT_SUBPROCESSES"], + :oncpu => ENV["PYROSCOPE_ONCPU"], + :version => ENV["RUBY_VERSION"] + } +end + +def work(n) + i = 0 + while i < n + i += 1 + end +end + +def fast_function + Pyroscope.tag_wrapper({"function": "fast"}) do + work(2001002000) + end +end + +def slow_function + work(8001008000) +end + +child_pid = fork do + puts "This is the child process" + Pyroscope.tag_wrapper({"fork": "forked"}) do + slow_function() + end +end + +puts "This is the master process." + +Pyroscope.tag_wrapper({"fork": "master"}) do + fast_function() +end + +puts "The PID of the child process is #{child_pid}" + +Pyroscope.shutdown() From b2faa2c26700b559e4bcd252f93e20980a11ded5 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 13:14:57 +0300 Subject: [PATCH 57/70] wip: initial ruby job --- .github/workflows/ci-ffi-ruby.yml | 305 +++++++++++++++++------------- 1 file changed, 169 insertions(+), 136 deletions(-) diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 1d6c6666..3c461337 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -15,7 +15,7 @@ on: - '**' jobs: - linux: + linux-build: strategy: fail-fast: false matrix: @@ -68,149 +68,182 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: ${{ github.sha }} + name: "linux.gem" path: pyroscope_ffi/ruby/pkg/*.gem - linux-arm: + linux-test: strategy: fail-fast: false matrix: - include: - - build-arch: aarch64 - target: aarch64-unknown-linux-gnu - - name: Linux - ${{ matrix.build-arch }} - runs-on: [self-hosted, Linux, ARM64] - - steps: - - uses: AutoModality/action-clean@v1 - - - uses: actions/checkout@v2 - - - name: Install bundles - run: bundle - working-directory: pyroscope_ffi/ruby - - - name: Update lock files - run: cargo update - working-directory: pyroscope_ffi/ruby/ext/rbspy - - - name: Update lock files - run: cargo update - working-directory: pyroscope_ffi/ruby/ext/thread_id - - - name: Build native extensions - run: rake thread_id_install - working-directory: pyroscope_ffi/ruby - - - name: Generate extra libraries - run: BUILD_ARCH=manylinux2014_aarch64 ./pyroscope_ffi/ruby/scripts/docker.sh - - - name: Copy generated extra libraries - run: mkdir -p lib/rbspy && cp ./elflib/rbspy/wheelhouse/rbspy-1.0.0/rbspy/_native__lib.cpython-37m-aarch64-linux-gnu.so lib/rbspy/rbspy.so - working-directory: pyroscope_ffi/ruby - - - name: Copy generated extra libraries - run: cp -r ./elflib/rbspy/wheelhouse/rbspy-1.0.0/rbspy.libs lib/rbspy.libs - working-directory: pyroscope_ffi/ruby - - - name: Build linux gem - run: rake aarch64_linux:gem - working-directory: pyroscope_ffi/ruby - - - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }} - path: pyroscope_ffi/ruby/pkg/*.gem - - macos: - strategy: - fail-fast: false - matrix: - include: - - macos-version: "11.0" - target: x86_64-apple-darwin - py-platform: macosx-11_0_x86_64 - - macos-version: "11.0" - target: aarch64-apple-darwin - py-platform: macosx-11_0_arm64 - - name: macOS - ${{ matrix.platform }} - runs-on: macos-${{ matrix.macos-version }} - - env: - RUST_TARGET: ${{ matrix.target }} - + PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + PYROSCOPE_ONCPU: [1, 0] + RUBY_VERSION: ['3.0'] + needs: ['linux-build'] + name: Linux Test + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: ${{ matrix.target }} - profile: minimal - override: true - - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.1' - - - name: Install bundles - run: bundle - working-directory: pyroscope_ffi/ruby - - - name: Update lock files - run: cargo update - working-directory: pyroscope_ffi/ruby/ext/rbspy - - - name: Update lock files - run: cargo update - working-directory: pyroscope_ffi/ruby/ext/thread_id - - - name: Build native extensions - run: rake rbspy_install - working-directory: pyroscope_ffi/ruby - - - name: Build native extensions - run: rake thread_id_install - working-directory: pyroscope_ffi/ruby - - - if: matrix.target == 'x86_64-apple-darwin' - name: Build macos gem - run: rake x86_64_darwin:gem - working-directory: pyroscope_ffi/ruby - - - if: matrix.target == 'aarch64-apple-darwin' - name: Build macos gem - run: rake arm64_darwin:gem - working-directory: pyroscope_ffi/ruby - - - uses: actions/upload-artifact@v2 + ruby-version: '3.0' + - uses: actions/download-artifact@master with: - name: ${{ github.sha }} - path: pyroscope_ffi/ruby/pkg/*.gem - - source: - name: source - runs-on: ubuntu-latest - - steps: - - uses: AutoModality/action-clean@v1 + name: "linux.gem" + path: "${{github.workspace}}/ruby" + - run: "cd ${{ github.workspace }}/ruby && ls -l" + - run: "cd ${{ github.workspace }}/ruby && gem install *.gem" - uses: actions/checkout@v2 - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.1' - - - name: Install bundles - run: bundle - working-directory: pyroscope_ffi/ruby - - - name: Build source gem - run: rake source:gem - working-directory: pyroscope_ffi/ruby - - - uses: actions/upload-artifact@v2 with: - name: ${{ github.sha }} - path: pyroscope_ffi/ruby/pkg/*.gem + submodules: recursive + - name: Run Ruby Script + run: pyroscope_ffi/ruby/scripts/tests/test.rb + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-linux + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + RUBY_VERSION: ${{ matrix.RUBY_VERSION }} + + #linux-arm: + #strategy: + #fail-fast: false + #matrix: + #include: + #- build-arch: aarch64 + #target: aarch64-unknown-linux-gnu + + #name: Linux - ${{ matrix.build-arch }} + #runs-on: [self-hosted, Linux, ARM64] + + #steps: + #- uses: AutoModality/action-clean@v1 + + #- uses: actions/checkout@v2 + + #- name: Install bundles + #run: bundle + #working-directory: pyroscope_ffi/ruby + + #- name: Update lock files + #run: cargo update + #working-directory: pyroscope_ffi/ruby/ext/rbspy + + #- name: Update lock files + #run: cargo update + #working-directory: pyroscope_ffi/ruby/ext/thread_id + + #- name: Build native extensions + #run: rake thread_id_install + #working-directory: pyroscope_ffi/ruby + + #- name: Generate extra libraries + #run: BUILD_ARCH=manylinux2014_aarch64 ./pyroscope_ffi/ruby/scripts/docker.sh + + #- name: Copy generated extra libraries + #run: mkdir -p lib/rbspy && cp ./elflib/rbspy/wheelhouse/rbspy-1.0.0/rbspy/_native__lib.cpython-37m-aarch64-linux-gnu.so lib/rbspy/rbspy.so + #working-directory: pyroscope_ffi/ruby + + #- name: Copy generated extra libraries + #run: cp -r ./elflib/rbspy/wheelhouse/rbspy-1.0.0/rbspy.libs lib/rbspy.libs + #working-directory: pyroscope_ffi/ruby + + #- name: Build linux gem + #run: rake aarch64_linux:gem + #working-directory: pyroscope_ffi/ruby + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/ruby/pkg/*.gem + + #macos: + #strategy: + #fail-fast: false + #matrix: + #include: + #- macos-version: "11.0" + #target: x86_64-apple-darwin + #py-platform: macosx-11_0_x86_64 + #- macos-version: "11.0" + #target: aarch64-apple-darwin + #py-platform: macosx-11_0_arm64 + + #name: macOS - ${{ matrix.platform }} + #runs-on: macos-${{ matrix.macos-version }} + + #env: + #RUST_TARGET: ${{ matrix.target }} + + #steps: + #- uses: actions/checkout@v2 + + #- uses: actions-rs/toolchain@v1 + #with: + #toolchain: stable + #target: ${{ matrix.target }} + #profile: minimal + #override: true + + #- uses: ruby/setup-ruby@v1 + #with: + #ruby-version: '3.1' + + #- name: Install bundles + #run: bundle + #working-directory: pyroscope_ffi/ruby + + #- name: Update lock files + #run: cargo update + #working-directory: pyroscope_ffi/ruby/ext/rbspy + + #- name: Update lock files + #run: cargo update + #working-directory: pyroscope_ffi/ruby/ext/thread_id + + #- name: Build native extensions + #run: rake rbspy_install + #working-directory: pyroscope_ffi/ruby + + #- name: Build native extensions + #run: rake thread_id_install + #working-directory: pyroscope_ffi/ruby + + #- if: matrix.target == 'x86_64-apple-darwin' + #name: Build macos gem + #run: rake x86_64_darwin:gem + #working-directory: pyroscope_ffi/ruby + + #- if: matrix.target == 'aarch64-apple-darwin' + #name: Build macos gem + #run: rake arm64_darwin:gem + #working-directory: pyroscope_ffi/ruby + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/ruby/pkg/*.gem + + #source: + #name: source + #runs-on: ubuntu-latest + + #steps: + #- uses: AutoModality/action-clean@v1 + #- uses: actions/checkout@v2 + + #- uses: ruby/setup-ruby@v1 + #with: + #ruby-version: '3.1' + + #- name: Install bundles + #run: bundle + #working-directory: pyroscope_ffi/ruby + + #- name: Build source gem + #run: rake source:gem + #working-directory: pyroscope_ffi/ruby + + #- uses: actions/upload-artifact@v2 + #with: + #name: ${{ github.sha }} + #path: pyroscope_ffi/ruby/pkg/*.gem From 538aa4c49562643b768d7933f0eb30708b162db4 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 13:30:23 +0300 Subject: [PATCH 58/70] wip: minor updates --- pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index ee6c5c4f..f92caec5 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -103,7 +103,7 @@ pub extern "C" fn initialize_logging(logging_level: u32) -> bool { #[no_mangle] pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, - sample_rate: u32, detect_subprocesses: bool, on_cpu: bool, report_pid: bool, + sample_rate: u32, detect_subprocesses: bool, oncpu: bool, report_pid: bool, report_thread_id: bool, tags: *const c_char, ) -> bool { // Initialize FFIKit @@ -134,8 +134,8 @@ pub extern "C" fn initialize_agent( let rbspy_config = RbspyConfig::new(pid.try_into().unwrap()) .sample_rate(sample_rate) .lock_process(false) - .with_subprocesses(detect_subprocesses) - .on_cpu(on_cpu) + .detect_subprocesses(detect_subprocesses) + .oncpu(oncpu) .report_pid(report_pid) .report_thread_id(report_thread_id); From 8a598b02af65c5837969c0a19c22c9c8c81bc969 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 13:44:26 +0300 Subject: [PATCH 59/70] wip: fix script bool --- pyroscope_ffi/ruby/scripts/tests/test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyroscope_ffi/ruby/scripts/tests/test.rb b/pyroscope_ffi/ruby/scripts/tests/test.rb index 1498af3f..c87a31a9 100755 --- a/pyroscope_ffi/ruby/scripts/tests/test.rb +++ b/pyroscope_ffi/ruby/scripts/tests/test.rb @@ -10,8 +10,8 @@ config.application_name = "new.fork.ruby" config.server_address = "https://ingest.pyroscope.cloud" config.auth_token = ENV["PYROSCOPE_API_TOKEN"] - config.detect_subprocesses = ENV["PYROSCOPE_DETECT_SUBPROCESSES"] - config.oncpu = ENV["PYROSCOPE_ONCPU"] + config.detect_subprocesses = ENV["PYROSCOPE_DETECT_SUBPROCESSES"] == "1" + config.oncpu = ENV["PYROSCOPE_ONCPU"] == "1" config.log_level = "trace" config.report_pid = true config.report_thread_id = true From ad02c2691e1787b6f6e4f7a0b86eadf9812fcc95 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 23:05:16 +0300 Subject: [PATCH 60/70] wip: change test name --- pyroscope_ffi/python/scripts/tests/test.py | 3 ++- pyroscope_ffi/ruby/scripts/tests/test.rb | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyroscope_ffi/python/scripts/tests/test.py b/pyroscope_ffi/python/scripts/tests/test.py index 5c7870fc..d57d3b05 100755 --- a/pyroscope_ffi/python/scripts/tests/test.py +++ b/pyroscope_ffi/python/scripts/tests/test.py @@ -13,7 +13,7 @@ # Configure Pyroscope pyroscope.configure( - application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}-{os.getenv("PYROSCOPE_ARCH")}-all', + application_name = f'{os.getenv("PYROSCOPE_RUN_ID")}', server_address = "https://ingest.pyroscope.cloud", auth_token = f'{os.getenv("PYROSCOPE_API_TOKEN")}', enable_logging =True, @@ -28,6 +28,7 @@ "oncpu": f'{os.getenv("PYROSCOPE_ONCPU")}', "gil_only": f'{os.getenv("PYROSCOPE_GIL_ONLY")}', "version": f'{os.getenv("PYTHON_VERSION")}', + "arch": f'{os.getenv("PYROSCOPE_ARCH")}', } ) diff --git a/pyroscope_ffi/ruby/scripts/tests/test.rb b/pyroscope_ffi/ruby/scripts/tests/test.rb index c87a31a9..33e9a466 100755 --- a/pyroscope_ffi/ruby/scripts/tests/test.rb +++ b/pyroscope_ffi/ruby/scripts/tests/test.rb @@ -7,7 +7,7 @@ puts RUBY_VERSION Pyroscope.configure do |config| - config.application_name = "new.fork.ruby" + config.application_name = "#{ENV["PYROSCOPE_RUN_ID"]}" config.server_address = "https://ingest.pyroscope.cloud" config.auth_token = ENV["PYROSCOPE_API_TOKEN"] config.detect_subprocesses = ENV["PYROSCOPE_DETECT_SUBPROCESSES"] == "1" @@ -19,7 +19,8 @@ :region => "us-east", :detect_subprocesses => ENV["PYROSCOPE_DETECT_SUBPROCESSES"], :oncpu => ENV["PYROSCOPE_ONCPU"], - :version => ENV["RUBY_VERSION"] + :version => ENV["RUBY_VERSION"], + :arch => ENV["PYROSCOPE_ARCH"] } end From 4b939c417a41c4647164d23201c5081ff57c361c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 23:08:03 +0300 Subject: [PATCH 61/70] wip: update ruby matrix --- .github/workflows/ci-ffi-ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 3c461337..f99ceafb 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -77,7 +77,7 @@ jobs: matrix: PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] PYROSCOPE_ONCPU: [1, 0] - RUBY_VERSION: ['3.0'] + RUBY_VERSION: ['2.6', '2.7', '3.0', '3.1'] needs: ['linux-build'] name: Linux Test runs-on: ubuntu-latest From 1087f24663ea2794564671613bbbc1db12cbb9ba Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 23:32:38 +0300 Subject: [PATCH 62/70] wip: ruby macos matrix --- .github/workflows/ci-ffi-ruby.yml | 141 ++++++++++++++++++------------ 1 file changed, 87 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index f99ceafb..ea4e3bcf 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -84,7 +84,7 @@ jobs: steps: - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.0' + ruby-version: ${{ matrix.RUBY_VERSION }} - uses: actions/download-artifact@master with: name: "linux.gem" @@ -156,72 +156,105 @@ jobs: #name: ${{ github.sha }} #path: pyroscope_ffi/ruby/pkg/*.gem - #macos: - #strategy: - #fail-fast: false - #matrix: - #include: - #- macos-version: "11.0" - #target: x86_64-apple-darwin - #py-platform: macosx-11_0_x86_64 - #- macos-version: "11.0" - #target: aarch64-apple-darwin - #py-platform: macosx-11_0_arm64 + macos: + strategy: + fail-fast: false + matrix: + include: + - macos-version: "11.0" + target: x86_64-apple-darwin + py-platform: macosx-11_0_x86_64 + - macos-version: "11.0" + target: aarch64-apple-darwin + py-platform: macosx-11_0_arm64 - #name: macOS - ${{ matrix.platform }} - #runs-on: macos-${{ matrix.macos-version }} + name: macOS - ${{ matrix.platform }} + runs-on: macos-${{ matrix.macos-version }} - #env: - #RUST_TARGET: ${{ matrix.target }} + env: + RUST_TARGET: ${{ matrix.target }} - #steps: - #- uses: actions/checkout@v2 + steps: + - uses: actions/checkout@v2 - #- uses: actions-rs/toolchain@v1 - #with: - #toolchain: stable - #target: ${{ matrix.target }} - #profile: minimal - #override: true + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: ${{ matrix.target }} + profile: minimal + override: true - #- uses: ruby/setup-ruby@v1 - #with: - #ruby-version: '3.1' + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.1' - #- name: Install bundles - #run: bundle - #working-directory: pyroscope_ffi/ruby + - name: Install bundles + run: bundle + working-directory: pyroscope_ffi/ruby - #- name: Update lock files - #run: cargo update - #working-directory: pyroscope_ffi/ruby/ext/rbspy + - name: Update lock files + run: cargo update + working-directory: pyroscope_ffi/ruby/ext/rbspy - #- name: Update lock files - #run: cargo update - #working-directory: pyroscope_ffi/ruby/ext/thread_id + - name: Update lock files + run: cargo update + working-directory: pyroscope_ffi/ruby/ext/thread_id - #- name: Build native extensions - #run: rake rbspy_install - #working-directory: pyroscope_ffi/ruby + - name: Build native extensions + run: rake rbspy_install + working-directory: pyroscope_ffi/ruby - #- name: Build native extensions - #run: rake thread_id_install - #working-directory: pyroscope_ffi/ruby + - name: Build native extensions + run: rake thread_id_install + working-directory: pyroscope_ffi/ruby - #- if: matrix.target == 'x86_64-apple-darwin' - #name: Build macos gem - #run: rake x86_64_darwin:gem - #working-directory: pyroscope_ffi/ruby + - if: matrix.target == 'x86_64-apple-darwin' + name: Build macos gem + run: rake x86_64_darwin:gem + working-directory: pyroscope_ffi/ruby - #- if: matrix.target == 'aarch64-apple-darwin' - #name: Build macos gem - #run: rake arm64_darwin:gem - #working-directory: pyroscope_ffi/ruby + - if: matrix.target == 'aarch64-apple-darwin' + name: Build macos gem + run: rake arm64_darwin:gem + working-directory: pyroscope_ffi/ruby - #- uses: actions/upload-artifact@v2 - #with: - #name: ${{ github.sha }} - #path: pyroscope_ffi/ruby/pkg/*.gem + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.target }} + path: pyroscope_ffi/ruby/pkg/*.gem + + macos-intel-test: + strategy: + fail-fast: false + matrix: + PYROSCOPE_DETECT_SUBPROCESSES: [1, 0] + PYROSCOPE_ONCPU: [1, 0] + RUBY_VERSION: ['2.6', '2.7', '3.0', '3.1'] + needs: ['macos-build'] + name: macOS Test + runs-on: macos-12.0 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.RUBY_VERSION }} + - uses: actions/download-artifact@master + with: + name: "linux.gem" + path: "${{github.workspace}}/ruby" + - run: "cd ${{ github.workspace }}/ruby && ls -l" + - run: "cd ${{ github.workspace }}/ruby && gem install *.gem" + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Run Ruby Script + run: pyroscope_ffi/ruby/scripts/tests/test.rb + env: + PYROSCOPE_RUN_ID: ${{ github.run_id }} + PYROSCOPE_ARCH: x86-64-apple-darwin + PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} + PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} + PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} + RUBY_VERSION: ${{ matrix.RUBY_VERSION }} #source: #name: source From c5c1f71f03af9864624ed3ec2eb64170942d9786 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Fri, 5 Aug 2022 23:33:19 +0300 Subject: [PATCH 63/70] wip: typo --- .github/workflows/ci-ffi-ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index ea4e3bcf..192b1bc0 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -156,7 +156,7 @@ jobs: #name: ${{ github.sha }} #path: pyroscope_ffi/ruby/pkg/*.gem - macos: + macos-build: strategy: fail-fast: false matrix: From a532e1131054bd5dea966c0055ae0c8ecb67eda2 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 6 Aug 2022 00:18:19 +0300 Subject: [PATCH 64/70] wip: change macos version --- .github/workflows/ci-ffi-python.yml | 4 ++-- .github/workflows/ci-ffi-ruby.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-ffi-python.yml b/.github/workflows/ci-ffi-python.yml index 13d29e6e..64a02f79 100644 --- a/.github/workflows/ci-ffi-python.yml +++ b/.github/workflows/ci-ffi-python.yml @@ -166,7 +166,7 @@ jobs: PYTHON_VERSION: ['2.7', '3.7', '3.8', '3.9', '3.10'] needs: ['macos-build'] name: Macos Test - runs-on: macos-12.0 + runs-on: macos-11.0 steps: - name: Set up Python uses: actions/setup-python@v4 @@ -189,7 +189,7 @@ jobs: run: pyroscope_ffi/python/scripts/tests/test.py env: PYROSCOPE_RUN_ID: ${{ github.run_id }} - PYROSCOPE_ARCH: x86-64-darwin + PYROSCOPE_ARCH: x86-64-apple-darwin PYROSCOPE_API_TOKEN: ${{ secrets.PYROSCOPE_API_TOKEN }} PYROSCOPE_DETECT_SUBPROCESSES: ${{ matrix.PYROSCOPE_DETECT_SUBPROCESSES }} PYROSCOPE_ONCPU: ${{ matrix.PYROSCOPE_ONCPU }} diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index 192b1bc0..fe96eefd 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -232,7 +232,7 @@ jobs: RUBY_VERSION: ['2.6', '2.7', '3.0', '3.1'] needs: ['macos-build'] name: macOS Test - runs-on: macos-12.0 + runs-on: macos-11.0 steps: - uses: ruby/setup-ruby@v1 with: From f9b729133c8058a4350ba7c3f6d34eb54bb12d7c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Sat, 6 Aug 2022 01:50:33 +0300 Subject: [PATCH 65/70] wip: fix typo --- .github/workflows/ci-ffi-ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ffi-ruby.yml b/.github/workflows/ci-ffi-ruby.yml index fe96eefd..584ac692 100644 --- a/.github/workflows/ci-ffi-ruby.yml +++ b/.github/workflows/ci-ffi-ruby.yml @@ -239,7 +239,7 @@ jobs: ruby-version: ${{ matrix.RUBY_VERSION }} - uses: actions/download-artifact@master with: - name: "linux.gem" + name: x86_64-apple-darwin path: "${{github.workspace}}/ruby" - run: "cd ${{ github.workspace }}/ruby && ls -l" - run: "cd ${{ github.workspace }}/ruby && gem install *.gem" From cdf23fd049f18b49c1a80a051564dfde942fcf11 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Sat, 6 Aug 2022 07:30:17 -0700 Subject: [PATCH 66/70] feat: upstream gzip compression (#55) * Tests actions (#53) * imp(regex): initial regex implementation * imp(ruby): add regex support * imp(lib): add report transformation function * imp(ruby): add report transformation function * chore(release): ruby gem 0.3.2 * ruby substitute function changes * refactor(examples): change regex example * imp(lib): add iter function to Report and StackTrace * refactor(session): minor refactoring * docs(README): update README files * imp(python): add enable_logging option * imp(ruby): add logging support * refactor(ruby): update ruby spec * chore(actions): renamed ruby/python ci files * wip: update action path * wip: test download artifact * wip: fix * wip: fix * wip: fix * wip: use env * wip: use github.workspace * wip: pip install * wip: sdist job * wip: cd into directory * wip: wrong place * wip: python test files * wip: add python script runner * wip: add run_id * wip: add other scripts * wip: arm tests * wip: add setup_python * wip: add macos tests * wip: install pip for arm * wip: add setup_python for macos * wip: use apt-get instead * wip: update pip * wip: minor * chore(cli): update dependencies * imp(pyspy): harmonize API * imp: harmonize API and bump versions * imp(cli): add auth_token/oncpu options * imp(cli): auth_token implementation * wip: single test file * wip: update ffi * wip: add env variables * wip: use an integer * wip: python bool * wip: python version matrix * wip: try pip3 * wip: yaml formatting * wip: use matrix for macos tests * wip: trying something * wip: get python version details * wip: change macos version * wip: remove test scripts * wip: ruby test file * wip: initial ruby job * wip: minor updates * wip: fix script bool * wip: change test name * wip: update ruby matrix * wip: ruby macos matrix * wip: typo * wip: change macos version * wip: fix typo Co-authored-by: Dmitry Filimonov * feat: add gzip compression * updates gem version, enables compression by default * small fixes Co-authored-by: Abid Omar Co-authored-by: Tolya Korniltsev --- Cargo.toml | 1 + pyroscope_ffi/ruby/ext/rbspy/src/lib.rs | 24 ++++++++-- pyroscope_ffi/ruby/lib/pyroscope.rb | 14 ++++-- pyroscope_ffi/ruby/lib/pyroscope/version.rb | 2 +- src/pyroscope.rs | 53 +++++++++++++++++++++ src/session.rs | 15 +++++- 6 files changed, 96 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 691874ce..7e0ea270 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ log = "0.4" names = "0.13.0" reqwest = { version = "0.11", features = ["blocking", "rustls-tls-native-roots"]} url = "2.2.2" +libflate = "1.2.0" libc = "^0.2.124" [dev-dependencies] diff --git a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs index f92caec5..8b0d3015 100644 --- a/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs +++ b/pyroscope_ffi/ruby/ext/rbspy/src/lib.rs @@ -1,12 +1,15 @@ -use ffikit::Signal; -use pyroscope::backend::{Report, StackFrame, Tag}; -use pyroscope::PyroscopeAgent; -use pyroscope_rbspy::{rbspy_backend, RbspyConfig}; use std::collections::hash_map::DefaultHasher; use std::env; use std::ffi::CStr; use std::hash::Hasher; use std::os::raw::c_char; +use std::str::FromStr; + +use ffikit::Signal; +use pyroscope_rbspy::{rbspy_backend, RbspyConfig}; + +use pyroscope::{pyroscope::Compression, PyroscopeAgent}; +use pyroscope::backend::{Report, StackFrame, Tag}; pub fn transform_report(report: Report) -> Report { let cwd = env::current_dir().unwrap(); @@ -104,7 +107,7 @@ pub extern "C" fn initialize_logging(logging_level: u32) -> bool { pub extern "C" fn initialize_agent( application_name: *const c_char, server_address: *const c_char, auth_token: *const c_char, sample_rate: u32, detect_subprocesses: bool, oncpu: bool, report_pid: bool, - report_thread_id: bool, tags: *const c_char, + report_thread_id: bool, tags: *const c_char, compression: *const c_char ) -> bool { // Initialize FFIKit let recv = ffikit::initialize_ffi().unwrap(); @@ -129,6 +132,13 @@ pub extern "C" fn initialize_agent( .unwrap() .to_string(); + let compression_string = unsafe { CStr::from_ptr(compression) } + .to_str() + .unwrap() + .to_string(); + + let compression = Compression::from_str(&compression_string); + let pid = std::process::id(); let rbspy_config = RbspyConfig::new(pid.try_into().unwrap()) @@ -152,6 +162,10 @@ pub extern "C" fn initialize_agent( agent_builder = agent_builder.auth_token(auth_token); } + if let Ok(compression) = compression { + agent_builder = agent_builder.compression(compression); + } + let agent = agent_builder.build().unwrap(); let agent_running = agent.start().unwrap(); diff --git a/pyroscope_ffi/ruby/lib/pyroscope.rb b/pyroscope_ffi/ruby/lib/pyroscope.rb index a43a8eb9..5371396f 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope.rb @@ -8,7 +8,7 @@ module Rust extend FFI::Library ffi_lib File.expand_path(File.dirname(__FILE__)) + "/rbspy/rbspy.#{RbConfig::CONFIG["DLEXT"]}" attach_function :initialize_logging, [:int], :bool - attach_function :initialize_agent, [:string, :string, :string, :int, :bool, :bool, :bool, :bool, :string], :bool + attach_function :initialize_agent, [:string, :string, :string, :int, :bool, :bool, :bool, :bool, :string, :string], :bool attach_function :add_thread_tag, [:uint64, :string, :string], :bool attach_function :remove_thread_tag, [:uint64, :string, :string], :bool attach_function :add_global_tag, [:string, :string], :bool @@ -22,8 +22,10 @@ module Utils attach_function :thread_id, [], :uint64 end - Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :log_level, :sample_rate, :detect_subprocesses, :oncpu, :report_pid, :report_thread_id, :tags) do + Config = Struct.new(:application_name, :app_name, :server_address, :auth_token, :log_level, :sample_rate, :detect_subprocesses, :oncpu, :report_pid, :report_thread_id, :tags, :compression) do def initialize(*) + super + # defaults: self.application_name = '' self.server_address = 'http://localhost:4040' self.auth_token = '' @@ -34,7 +36,7 @@ def initialize(*) self.report_thread_id = false self.log_level = 'error' self.tags = {} - super + self.compression = 'gzip' end end @@ -63,10 +65,11 @@ def configure # Initialize Logging Rust.initialize_logging(@log_level) - + # initialize Pyroscope Agent Rust.initialize_agent( + # these are defaults in case user-provided values are nil: @config.app_name || @config.application_name || "", @config.server_address || "", @config.auth_token || "", @@ -75,7 +78,8 @@ def configure @config.oncpu || false, @config.report_pid || false, @config.report_thread_id || false, - tags_to_string(@config.tags || {}) + tags_to_string(@config.tags || {}), + @config.compression || "" ) end diff --git a/pyroscope_ffi/ruby/lib/pyroscope/version.rb b/pyroscope_ffi/ruby/lib/pyroscope/version.rb index 445cf038..6fc22b1c 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope/version.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope/version.rb @@ -1,3 +1,3 @@ module Pyroscope - VERSION = '0.3.2'.freeze + VERSION = '0.4.0'.freeze end diff --git a/src/pyroscope.rs b/src/pyroscope.rs index 32edae81..f286c4f9 100644 --- a/src/pyroscope.rs +++ b/src/pyroscope.rs @@ -1,6 +1,7 @@ use std::{ collections::HashMap, marker::PhantomData, + str::FromStr, sync::{ mpsc::{self, Sender}, Arc, Condvar, Mutex, @@ -18,6 +19,7 @@ use crate::{ }; use crate::backend::BackendImpl; +use crate::pyroscope::Compression::GZIP; const LOG_TAG: &str = "Pyroscope::Agent"; @@ -44,6 +46,8 @@ pub struct PyroscopeConfig { pub auth_token: Option, /// Function to apply pub func: Option Report>, + /// Pyroscope http request body compression + pub compression: Option, } impl Default for PyroscopeConfig { @@ -59,6 +63,7 @@ impl Default for PyroscopeConfig { spy_name: "undefined".to_string(), auth_token: None, func: None, + compression: None, } } } @@ -80,6 +85,7 @@ impl PyroscopeConfig { spy_name: String::from("undefined"), // Spy Name should be set by the backend auth_token: None, // No authentication token func: None, // No function + compression: None, } } @@ -150,6 +156,22 @@ impl PyroscopeConfig { ..self } } + + + /// Set the http request body compression. + /// + /// # Example + /// ```ignore + /// use pyroscope::pyroscope::PyroscopeConfig; + /// let config = PyroscopeConfig::new("http://localhost:8080", "my-app") + /// .compression(GZIP); + /// ``` + pub fn compression(self, compression: Compression) -> Self { + Self { + compression: Some(compression), + ..self + } + } } /// PyroscopeAgent Builder @@ -288,6 +310,22 @@ impl PyroscopeAgentBuilder { } } + /// Set the http request body compression. + /// + /// # Example + /// ```ignore + /// use pyroscope::pyroscope::PyroscopeConfig; + /// let agent = PyroscopeAgentBuilder::new("http://localhost:8080", "my-app") + /// .compression(GZIP) + /// .build(); + /// ``` + pub fn compression(self, compression: Compression) -> Self { + Self { + config: self.config.compression(compression), + ..self + } + } + /// Initialize the backend, timer and return a PyroscopeAgent with Ready /// state. While you can call this method, you should call it through the /// `PyroscopeAgent.build()` method. @@ -344,6 +382,21 @@ impl PyroscopeAgentBuilder { } } +#[derive(Clone, Debug)] +pub enum Compression { + GZIP +} + +impl FromStr for Compression { + type Err = (); + fn from_str(input: &str) -> std::result::Result { + match input { + "gzip" => Ok(GZIP), + _ => Err(()), + } + } +} + /// This trait is used to encode the state of the agent. pub trait PyroscopeAgentState {} diff --git a/src/session.rs b/src/session.rs index 15e79e1d..71994d04 100644 --- a/src/session.rs +++ b/src/session.rs @@ -2,13 +2,15 @@ use std::{ sync::mpsc::{sync_channel, Receiver, SyncSender}, thread::{self, JoinHandle}, time::Duration, + io::Write, }; use reqwest::Url; +use libflate::gzip::Encoder; use crate::{ backend::Report, - pyroscope::PyroscopeConfig, + pyroscope::{PyroscopeConfig, Compression}, utils::{get_time_range, merge_tags_with_app_name}, Result, }; @@ -192,6 +194,15 @@ impl Session { if let Some(auth_token) = self.config.auth_token.clone() { req_builder = req_builder.bearer_auth(auth_token); } + let body = match &self.config.compression { + None => report_u8, + Some(Compression::GZIP) => { + req_builder = req_builder.header("Content-Encoding", "gzip"); + let mut encoder = Encoder::new(Vec::new()).unwrap(); + encoder.write_all(&report_u8).unwrap(); + encoder.finish().into_result().unwrap() + } + }; // Send the request req_builder @@ -203,7 +214,7 @@ impl Session { ("sampleRate", &format!("{}", self.config.sample_rate)), ("spyName", self.config.spy_name.as_str()), ]) - .body(report_u8) + .body(body) .timeout(Duration::from_secs(10)) .send()?; Ok(()) From 160fa82a69ec9e773e3575ed24b27a5b2987df2c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Tue, 9 Aug 2022 11:27:49 +0300 Subject: [PATCH 67/70] wip(lib): minor refactoring --- CHANGELOG.md | 11 ++++++++++- pyroscope_ffi/ffikit/src/lib.rs | 19 ++++++++++++++++--- src/backend/backend.rs | 13 +------------ src/session.rs | 2 +- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d95e21f..2b89930e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ +# v0.5.4 +## New Features +- Add report transfromation function which allows changing the report before + sending it to the Pyroscope Server. +- Add Gzip support. + +## Bug Fixes +- Use URL Builder. ([786c89b](https://github.com/pyroscope-io/pyroscope-rs/commit/786c89bb99839c45778410012a6da60267d395df)) + # v0.5.3 -## New features +## New Features - Add BackendConfig to make reporting of pid, thread_id and thread_name optional. - Backends can add a suffix to the "application_name" diff --git a/pyroscope_ffi/ffikit/src/lib.rs b/pyroscope_ffi/ffikit/src/lib.rs index b0f73f92..cb2659a8 100644 --- a/pyroscope_ffi/ffikit/src/lib.rs +++ b/pyroscope_ffi/ffikit/src/lib.rs @@ -23,9 +23,13 @@ static ONCE: Once = Once::new(); lazy_static! { /// Root Sender + /// This is the sender to the main loop. It is lazily initialized inside a Mutex. static ref SENDER: Mutex>> = Mutex::new(None); } +/// Signal enum. +/// This enum is used to send signals to the main loop. It is used to add/remove global or thread +/// tags and to exit the main loop. #[derive(Debug, Encode, Decode, PartialEq, Clone)] pub enum Signal { Kill, @@ -52,7 +56,7 @@ pub fn initialize_ffi() -> Result> { // Listen for signals on the main parent process. let fn_sender = merge_sender.clone(); - let channel_listener: JoinHandle> = std::thread::spawn(move || { + let _channel_listener: JoinHandle> = std::thread::spawn(move || { log::trace!("Spawned FFI listener thread."); while let Ok(signal) = receiver.recv() { @@ -81,7 +85,7 @@ pub fn initialize_ffi() -> Result> { // Listen for signals on local socket let socket_sender = merge_sender.clone(); - let socket_listener: JoinHandle> = std::thread::spawn(move || { + let _socket_listener: JoinHandle> = std::thread::spawn(move || { let socket_address = format!("/tmp/PYROSCOPE-{}", get_parent_pid()); log::trace!( @@ -158,7 +162,16 @@ pub fn send(signal: Signal) -> Result<()> { conn.flush()?; } else { // Send signal through parent process. - SENDER.lock()?.as_ref().unwrap().send(signal)?; + if let Some(sender) = &*SENDER.lock()? { + log::trace!( + target: LOG_TAG, + "Sending signal {:?} through FFI channel", + signal + ); + sender.send(signal)?; + } else { + log::error!(target: LOG_TAG, "FFI channel not initialized"); + } } Ok(()) diff --git a/src/backend/backend.rs b/src/backend/backend.rs index cd9dada3..1e51e2d2 100644 --- a/src/backend/backend.rs +++ b/src/backend/backend.rs @@ -12,7 +12,7 @@ use std::{ use super::Report; /// Backend Config -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Default)] pub struct BackendConfig { pub report_thread_id: bool, pub report_thread_name: bool, @@ -20,17 +20,6 @@ pub struct BackendConfig { pub report_oncpu: bool, } -impl Default for BackendConfig { - fn default() -> Self { - Self { - report_thread_id: false, - report_thread_name: false, - report_pid: false, - report_oncpu: false, - } - } -} - /// Backend Trait pub trait Backend: Send + Debug { /// Backend Spy Name diff --git a/src/session.rs b/src/session.rs index 15e79e1d..f48ff14e 100644 --- a/src/session.rs +++ b/src/session.rs @@ -156,7 +156,7 @@ impl Session { let mut report_owned = report.to_owned(); // Apply function to the report - if let Some(func) = self.config.func.clone() { + if let Some(func) = self.config.func { report_owned = func(report_owned); } From 0b832805a3008aa99a20c672452440bb5ee8855a Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Tue, 9 Aug 2022 12:35:05 +0300 Subject: [PATCH 68/70] refactor(examples): update examples --- examples/internal/pyspy-connect.rs | 4 ++-- examples/internal/rbspy-connect.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/internal/pyspy-connect.rs b/examples/internal/pyspy-connect.rs index c590ac88..d9f7420b 100644 --- a/examples/internal/pyspy-connect.rs +++ b/examples/internal/pyspy-connect.rs @@ -20,8 +20,8 @@ fn main() -> Result<()> { .sample_rate(100) .lock_process(true) .time_limit(None) - .with_subprocesses(true) - .include_idle(false) + .detect_subprocesses(true) + .oncpu(false) .native(false); let agent = PyroscopeAgent::builder("http://localhost:4040", "pyspy.basic") diff --git a/examples/internal/rbspy-connect.rs b/examples/internal/rbspy-connect.rs index e810381b..5ae18431 100644 --- a/examples/internal/rbspy-connect.rs +++ b/examples/internal/rbspy-connect.rs @@ -19,7 +19,7 @@ fn main() -> Result<()> { let config = RbspyConfig::new(pid) .sample_rate(100) .lock_process(true) - .with_subprocesses(true); + .detect_subprocesses(true); let agent = PyroscopeAgent::builder("http://localhost:4040", "rbspy.basic") .tags([("Host", "Ruby")].to_vec()) From 6608943330f1a6ae33b63e902a61a90bbc9dec72 Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 15 Aug 2022 09:36:43 +0300 Subject: [PATCH 69/70] refactor(timer): remove warning --- src/timer/epoll.rs | 15 +++++++++------ src/timer/kqueue.rs | 15 +++++++++------ src/timer/sleep.rs | 15 +++++++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/timer/epoll.rs b/src/timer/epoll.rs index 0cf2f275..c5abf8c7 100644 --- a/src/timer/epoll.rs +++ b/src/timer/epoll.rs @@ -76,12 +76,15 @@ impl Timer { Ok(_) => { log::trace!(target: LOG_TAG, "Sent event to listener @ {:?}", &tx) } - Err(e) => log::warn!( - target: LOG_TAG, - "Failed to send event to listener @ {:?} - {}", - &tx, - e - ), + Err(_e) => {} + // There could be a less confusing message, or this + // refactored to avoid a first sender + //log::warn!( + //target: LOG_TAG, + //"Failed to send event to listener @ {:?} - {}", + //&tx, + //e + //), } }); } diff --git a/src/timer/kqueue.rs b/src/timer/kqueue.rs index 7b3adb00..5420738e 100644 --- a/src/timer/kqueue.rs +++ b/src/timer/kqueue.rs @@ -74,12 +74,15 @@ impl Timer { Ok(_) => { log::trace!(target: LOG_TAG, "Sent event to listener @ {:?}", &tx) } - Err(e) => log::warn!( - target: LOG_TAG, - "Failed to send event to listener @ {:?} - {}", - &tx, - e - ), + Err(_e) => {} + // There could be a less confusing message, or this + // refactored to avoid a first sender + //log::warn!( + //target: LOG_TAG, + //"Failed to send event to listener @ {:?} - {}", + //&tx, + //e + //), } }); diff --git a/src/timer/sleep.rs b/src/timer/sleep.rs index 8a44a58f..439c7315 100644 --- a/src/timer/sleep.rs +++ b/src/timer/sleep.rs @@ -72,12 +72,15 @@ impl Timer { Ok(_) => { log::trace!(target: LOG_TAG, "Sent event to listener @ {:?}", &tx) } - Err(e) => log::warn!( - target: LOG_TAG, - "Failed to send event to listener @ {:?} - {}", - &tx, - e - ), + Err(_e) => {} + // There could be a less confusing message, or this + // refactored to avoid a first sender + //log::warn!( + //target: LOG_TAG, + //"Failed to send event to listener @ {:?} - {}", + //&tx, + //e + //), } }); From 6abac6723842d811e0c39fa55ce96a420ad8419c Mon Sep 17 00:00:00 2001 From: Abid Omar Date: Mon, 15 Aug 2022 09:38:02 +0300 Subject: [PATCH 70/70] chore(release): bump ruby/python versions --- pyroscope_ffi/python/setup.cfg | 2 +- pyroscope_ffi/ruby/lib/pyroscope/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyroscope_ffi/python/setup.cfg b/pyroscope_ffi/python/setup.cfg index e9fca769..5fe09710 100644 --- a/pyroscope_ffi/python/setup.cfg +++ b/pyroscope_ffi/python/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyroscope-io -version= 0.7.2 +version= 0.7.3 description = Pyroscope Python integration long_description = file: README.md long_description_content_type = text/markdown diff --git a/pyroscope_ffi/ruby/lib/pyroscope/version.rb b/pyroscope_ffi/ruby/lib/pyroscope/version.rb index 6fc22b1c..58bc2d1b 100644 --- a/pyroscope_ffi/ruby/lib/pyroscope/version.rb +++ b/pyroscope_ffi/ruby/lib/pyroscope/version.rb @@ -1,3 +1,3 @@ module Pyroscope - VERSION = '0.4.0'.freeze + VERSION = '0.4.1'.freeze end