From 06395b782abdf0e4cb2aafea185baec4215b467c Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 31 Oct 2023 15:57:38 -0400 Subject: [PATCH] common: introduce cflow-based call stacks analysis utilities It complements an already existing stack usage extraction tool. Signed-off-by: Jan Michalski --- .github/workflows/scan_bandit.yml | 8 +- utils/call_stacks_analysis/README.md | 71 ++ utils/call_stacks_analysis/api_callers.py | 45 ++ .../examples/call_stack.json | 15 + .../call_stacks_analysis/examples/config.json | 19 + .../generate_call_stacks.py | 301 ++++++++ .../call_stacks_analysis/libpmem/config.json | 520 ++++++++++++++ .../libpmem/make_config.py | 186 +++++ .../libpmem/white_list.json | 304 +++++++++ .../libpmemobj/config.json | 645 ++++++++++++++++++ .../libpmemobj/make_config.py | 303 ++++++++ .../libpmemobj/white_list.json | 318 +++++++++ utils/call_stacks_analysis/stack_usage.py | 49 ++ .../stack_usage_stats.sh} | 3 +- 14 files changed, 2781 insertions(+), 6 deletions(-) create mode 100644 utils/call_stacks_analysis/README.md create mode 100755 utils/call_stacks_analysis/api_callers.py create mode 100644 utils/call_stacks_analysis/examples/call_stack.json create mode 100644 utils/call_stacks_analysis/examples/config.json create mode 100755 utils/call_stacks_analysis/generate_call_stacks.py create mode 100644 utils/call_stacks_analysis/libpmem/config.json create mode 100755 utils/call_stacks_analysis/libpmem/make_config.py create mode 100644 utils/call_stacks_analysis/libpmem/white_list.json create mode 100644 utils/call_stacks_analysis/libpmemobj/config.json create mode 100755 utils/call_stacks_analysis/libpmemobj/make_config.py create mode 100644 utils/call_stacks_analysis/libpmemobj/white_list.json create mode 100755 utils/call_stacks_analysis/stack_usage.py rename utils/{stack-usage-stats.sh => call_stacks_analysis/stack_usage_stats.sh} (74%) diff --git a/.github/workflows/scan_bandit.yml b/.github/workflows/scan_bandit.yml index 82d035acbea..d803661a766 100644 --- a/.github/workflows/scan_bandit.yml +++ b/.github/workflows/scan_bandit.yml @@ -5,9 +5,9 @@ on: workflow_call: env: - # Set path to the pmreorder tool. At the moment pmreorder is the only - # Python-based tool released in the PMDK. - SCAN_DIR: src/tools/pmreorder + # Python-based tools. + PMREORDER: src/tools/pmreorder/*.py + CALL_STACKS_ANALYSIS: utils/call_stacks_analysis/*.py jobs: bandit: @@ -21,4 +21,4 @@ jobs: run: sudo apt-get -y install bandit - name: Bandit scan - run: bandit --version && bandit -r "$SCAN_DIR" + run: bandit --version && bandit $PMREORDER $CALL_STACKS_ANALYSIS diff --git a/utils/call_stacks_analysis/README.md b/utils/call_stacks_analysis/README.md new file mode 100644 index 00000000000..c130b2a9fdb --- /dev/null +++ b/utils/call_stacks_analysis/README.md @@ -0,0 +1,71 @@ +# Call-stacks analysis utilities + +> XXX This document requires more details. + +1. `stack_usage_stats.sh` generate `src/stats/stack-usage-$build.txt` files. +2. Collect [cflow](https://savannah.gnu.org/git/?group=cflow) data. +3. Generate all possible call stacks given the data provided. + +```sh +# -u, --stack-usage-stat-file +# -f, --cflow-output-file +# -i, --config-file +./utils/call_stacks_analysis/generate_call_stacks.py \ + -u src/stats/stack-usage-nondebug.txt \ + -f src/libpmem/cflow.txt \ + -i utils/call_stacks_analysis/libpmem/config.json +``` + +If succesfull, it produces: + +- `call_stacks_all.json` with call stacks ordered descending by call stack consumption. +- `stack_usage.json` with the data extracted from the provided `src/stats/stack-usage-nondebug.txt` but limited to a single library according to the `config.json` filter value. + +**Note**: If too many functions ought to be added to a white list it might be useful to ignore functions having a certain stack usage or lower. Please see `-t` option to set a desired threshold. + +4. (Optional) Break down a call stack's stack consumption per function. Use the `stack_usage.json` as produced in the previous step and extract a single call stack and put it into a file (name `call_stack.json` below). Please see the examples directory for an example. + +```sh +# -s, --stack-usage-file +# -c, --call-stack +./utils/call_stacks_analysis/stack_usage.py \ + -s stack_usage.json \ + -c call_stack.json +``` + +If successful, it prints out on the screen a list of functions along with their stack consumption e.g. + +``` +208 pmem_map_file +0 pmem_map_fileU +80 pmem_map_register +64 util_range_register +240 util_ddax_region_find +8224 pmem2_get_type_from_stat +0 ERR +384 out_err +0 out_error +224 out_snprintf +``` + +5. (Optional) List all API calls which call stacks contains a given function. Use the `stack_usage.json` as produced in the previous step. + +```sh +# -a, --all-call-stacks-file +# -f, --function-name +./utils/call_stacks_analysis/api_callers.py \ + -a call_stacks_all.json \ + -f pmem2_get_type_from_stat +``` + +If successful, it prints out on screen a list of API calls that met the condition e.g. + +``` +os_part_deep_common +pmem_map_file +util_fd_get_type +util_file_device_dax_alignment +util_file_pread +util_file_pwrite +util_unlink_flock +``` diff --git a/utils/call_stacks_analysis/api_callers.py b/utils/call_stacks_analysis/api_callers.py new file mode 100755 index 00000000000..5b69e05a239 --- /dev/null +++ b/utils/call_stacks_analysis/api_callers.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +import argparse +import json + +from typing import List, Dict, Any + +# https://peps.python.org/pep-0589/ requires Python >= 3.8 +# from typing import TypedDict + +# List all API calls which start call stacks containing a particular function name. + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('-a', '--all-call-stacks-file', required=True) +PARSER.add_argument('-f', '--function-name', required=True) + +# class CallStack(TypedDict): # for Python >= 3.8 +# stack: list[str] +# size: int +CallStack = Dict[str, Any] # for Python < 3.8 + +def load_all_call_stacks(all_call_stacks_file: str) -> List[CallStack]: + with open(all_call_stacks_file, 'r') as file: + return json.load(file) + +def main(): + args = PARSER.parse_args() + call_stacks = load_all_call_stacks(args.all_call_stacks_file) + apis = [] + # lookup all call stacks in which the function of interest is mentioned + for call_stack in call_stacks: + if args.function_name in call_stack['stack']: + # callect all API calls which starts these call stacks + apis.append(call_stack['stack'][0]) + # remove duplicates + apis = list(set(apis)) + apis.sort() + for api in apis: + print(api) + +if __name__ == '__main__': + main() diff --git a/utils/call_stacks_analysis/examples/call_stack.json b/utils/call_stacks_analysis/examples/call_stack.json new file mode 100644 index 00000000000..a107e769ff7 --- /dev/null +++ b/utils/call_stacks_analysis/examples/call_stack.json @@ -0,0 +1,15 @@ +{ + "stack": [ + "pmem_map_file", + "pmem_map_fileU", + "pmem_map_register", + "util_range_register", + "util_ddax_region_find", + "pmem2_get_type_from_stat", + "ERR", + "out_err", + "out_error", + "out_snprintf" + ], + "size": 9424 + } diff --git a/utils/call_stacks_analysis/examples/config.json b/utils/call_stacks_analysis/examples/config.json new file mode 100644 index 00000000000..6f7491de3d7 --- /dev/null +++ b/utils/call_stacks_analysis/examples/config.json @@ -0,0 +1,19 @@ +{ + "__comment": "only consider stack usage for functions listed in the src/(non)debug// directory", + "filter": "library_name", + "api": [ + "api_call_1" + ], + "dead_end": [ + "not_called_function_1" + ], + "extra_calls": { + "caller_1": [ + "callee_1", + "callee_2" + ] + }, + "white_list": [ + "irrelevant_function_1" + ] +} \ No newline at end of file diff --git a/utils/call_stacks_analysis/generate_call_stacks.py b/utils/call_stacks_analysis/generate_call_stacks.py new file mode 100755 index 00000000000..97a9eded39f --- /dev/null +++ b/utils/call_stacks_analysis/generate_call_stacks.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +import argparse +import json +import re + +from typing import List, Dict, Any + +# https://peps.python.org/pep-0589/ requires Python >= 3.8 +# from typing import TypedDict +# https://peps.python.org/pep-0613/ requires Python >= 3.10 +# from typing import TypeAlias + + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('-u', '--stack-usage-stat-file', required=True) +PARSER.add_argument('-f', '--cflow-output-file', required=True) +PARSER.add_argument('-i', '--config-file', required=True) +PARSER.add_argument('-d', '--dump', action='store_true', help='Dump debug files') +PARSER.add_argument('-t', '--skip-threshold', type=int, default=0, + help='Ignore non-reachable function if its stack usage <= threshold') + +# class Config(TypedDict): # for Python >= 3.8 +# filter: str +# api: list[str] +# dead_end: list[str] +# extra_calls: dict[str, list[str]] +# white_list: list[str] +Config = Dict[str, Any] # for Python < 3.8 + +# class StackUsageRecord(TypedDict): # for Python >= 3.8 +# size: int +# type: str +StackUsageRecord = Dict[str, Any] # for Python < 3.8 + +# StackUsage: TypeAlias = dict[str, StackUsageRecord] # for Python >= 3.10 +StackUsage = Dict[str, StackUsageRecord] # for Python < 3.10 + +# Calls: TypeAlias = dict[str, list[str]] # for Python >= 3.10 +Calls = Dict[str, List[str]] # for Python < 3.10 + +# RCalls: TypeAlias = Calls # for Python >= 3.10 +RCalls = Calls # for Python < 3.10 + +# class CallStack(TypedDict): # for Python >= 3.8 +# stack: list[str] +# size: int +CallStack = Dict[str, Any] # for Python < 3.8 + +DUMP = False + +def dump(var, name: str, force: bool = False) -> None: + global DUMP + if not force and not DUMP: + return + with open(f'{name}.json', 'w') as outfile: + json.dump(var, outfile, indent = 4) + +def load_config(config_file: str) -> Config: + with open(config_file, 'r') as file: + return json.load(file) + +def parse_stack_usage(stack_usage_stat_file: str, filter: str) -> StackUsage: + funcs = {} + with open(stack_usage_stat_file, 'r') as file: + for line in file: + # 8432 out_common : src/nondebug/libpmem/out.su:out.c dynamic,bounded + found = re.search('([0-9]+) ([a-zA-Z0-9_]+)(.[a-z0-9.]+)* : ([a-z0-9.:/_-]+) ([a-z,]+)', line) + if found: + if filter in found.group(4): + funcs[found.group(2)] = {'size': int(found.group(1)), 'type': found.group(5)} + else: + print(f'An unexpected line format: {line}') + exit(1) + return funcs + +def parse_cflow_output(cflow_output_file: str) -> Calls: + calls = {} + call_stack = [] + with open(cflow_output_file, 'r') as file: + # processing line-by-line + for line in file: + line_copy = line + # determine the level of nesting + level = 0 + while line[0] == ' ': + level += 1 + line = line[4:] # remove the prepended spaces +# pmem_memset_persist() : + found = re.search('^([a-zA-Z0-9_]+)\(\)', line) + if not found: + print('An unexpected line format:') + print(line_copy) + exit(1) + func = found.group(1) + # construct the call stack being currently processed + call_stack.insert(level, func) + # being level 0 it does not have a caller + if level == 0: + continue + callee = func + caller = call_stack[level - 1] + if caller in calls.keys(): + calls[caller].append(callee) + else: + calls[caller] = [callee] + # remove duplicate callees + calls_unique = {} + for k, v in calls.items(): + v_unique = list(set(v)) + calls_unique[k] = v_unique + return calls_unique + +def dict_extend(dict_, key, values): + if key not in dict_.keys(): + dict_[key] = values + else: + dict_[key].extend(values) + return dict_ + +def include_extra_calls(calls: Calls, config: Config) -> Calls: + for k, v in config['extra_calls'].items(): + if k not in calls.keys(): + calls[k] = v + else: + calls[k].extend(v) + return calls + +def find_api_callers(func: str, calls: Calls, config: Config): + callers = [func] + visited = [func] # loop breaker + apis = [] + while len(callers) > 0: + callers_new = [] + for callee in callers: + for k, v in calls.items(): + # this caller does not call this callee + if callee not in v: + continue + # it is part of the API + if k in visited: + continue + if k in config['api'] or k in config['dead_end']: + apis.append(k) + else: + callers_new.append(k) + visited.append(k) + callers = list(set(callers_new)) + # print(callers) + # if len(apis) > 0 and len(callers) > 0: + # exit(1) + # if len(apis) == 0: + # print(func) + # assert(len(apis) > 0) + return apis + +def validate(stack_usage: StackUsage, calls: Calls, config: Config, skip_threshold: int) -> None: + all_callees = [] + for _, v in calls.items(): + all_callees.extend(v) + all_callees = list(set(all_callees)) + dump(all_callees, 'all_callees') + + # all known functions are expected to be called at least once + not_called = [] + for k, v in stack_usage.items(): + if k in all_callees: + continue + if k in config['api']: + continue + if k in config['dead_end']: + continue + if k in config['white_list']: + continue + if v['size'] <= skip_threshold: + continue + not_called.append(k) + # Use --dump to see the list of not called functions. + # Investigate and either fix the call graph or add it to the white list. + dump(not_called, 'not_called') + assert(len(not_called) == 0) + + # all known functions are expected to be reachable from the API + no_api_connection = {} + for k, v in stack_usage.items(): + if k in config['api']: + continue + if k in config['dead_end']: + continue + if k in config['white_list']: + continue + if v['size'] <= skip_threshold: + continue + callers = find_api_callers(k, calls, config) + if len(callers) == 0: + no_api_connection[k] = v['size'] + dump(no_api_connection, 'no_api_connection') + assert(len(no_api_connection) == 0) + +def prepare_rcalls(calls: Calls) -> Calls: + # preparing a reverse call dictionary + rcalls: dict[str, list] = {} + for caller, callees in calls.items(): + for callee in callees: + if callee in rcalls.keys(): + rcalls[callee].append(caller) + else: + rcalls[callee] = [caller] + dump(rcalls, 'rcalls') + return rcalls + +def generate_call_stacks(func: str, stack_usage: StackUsage, calls: Calls, rcalls: RCalls, config: Config) -> List[CallStack]: + call_stacks = [ + { + 'stack': [func], + 'size': int(stack_usage[func]['size']) if func in stack_usage.keys() else 0 + } + ] + # call stack generation loop + while True: + call_stacks_new = [] + # list of call stacks which cannot grow any more + call_stacks_new_end = [] + for call_stack in call_stacks: + callee = call_stack['stack'][0] + if callee in config['api']: + call_stacks_new_end.append(call_stack) + continue + if callee not in rcalls.keys(): + call_stacks_new_end.append(call_stack) + continue + for caller in rcalls[callee]: + # Note: Breaking the loop does not spoil generating + # other call stacks spawning from the same stem. + if call_stack['stack'].count(caller) == 2: + continue # loop breaker + if caller in stack_usage.keys(): + caller_stack_size = int(stack_usage[caller]['size']) + else: + caller_stack_size = 0 + call_stacks_new.append({ + 'stack': [caller] + call_stack['stack'], + 'size': call_stack['size'] + caller_stack_size + }) + if len(call_stacks_new) == 0: + break + call_stacks = call_stacks_new + call_stacks_new_end + return call_stacks + +def call_stack_key(e): + return e['size'] + +def generate_all_call_stacks(stack_usage: StackUsage, calls: Calls, rcalls: RCalls, config: Config, debug: bool = False) -> List[CallStack]: + call_stacks = [] + # loop over called functions + for func in rcalls.keys(): + if func in config['white_list']: + continue + # if a function calls something else, call stack generation will start from its callees + if func in calls.keys(): + continue + if debug: + print(f'Generating call stacks ending at - {func}') + call_stacks.extend(generate_call_stacks(func, stack_usage, calls, rcalls, config)) + call_stacks.sort(reverse=True, key=call_stack_key) + return call_stacks + +def main(): + args = PARSER.parse_args() + global DUMP + DUMP = args.dump # pass the argument value to a global variable + + config = load_config(args.config_file) + print('Load config - done') + + stack_usage = parse_stack_usage(args.stack_usage_stat_file, config['filter']) + # dumping stack_usage.json to allow further processing + dump(stack_usage, 'stack_usage', True) + print('Stack usage - done') + + calls = parse_cflow_output(args.cflow_output_file) + calls = include_extra_calls(calls, config) + dump(calls, 'calls') + print('Function calls - done') + + validate(stack_usage, calls, config, args.skip_threshold) + print('Validation - done') + + rcalls = prepare_rcalls(calls) + print('Reverse calls - done') + + call_stacks = generate_all_call_stacks(stack_usage, calls, rcalls, config) + dump(call_stacks, 'call_stacks_all', True) + print('Number of found call stacks: {}'.format(len(call_stacks))) + print('Call stack generation - done') + +if __name__ == '__main__': + main() diff --git a/utils/call_stacks_analysis/libpmem/config.json b/utils/call_stacks_analysis/libpmem/config.json new file mode 100644 index 00000000000..e8cf64ce862 --- /dev/null +++ b/utils/call_stacks_analysis/libpmem/config.json @@ -0,0 +1,520 @@ +{ + "filter": "libpmem/", + "api": [ + "pmem_map_file", + "pmem_unmap", + "pmem_memset", + "pmem_memmove", + "pmem_memcpy", + "pmem_memset_persist", + "pmem_memmove_persist", + "pmem_memcpy_persist", + "pmem_memcpy_nodrain", + "pmem_deep_persist", + "pmem_persist", + "pmem_check_version", + "libpmem_init", + "libpmem_fini", + "pmem_has_hw_drain", + "pmem_has_auto_flush", + "pmem_errormsg", + "pmem_memset_persist", + "pmem_memmove_persist" + ], + "dead_end": [], + "extra_calls": { + "core_init": [ + "util_init", + "out_init" + ], + "core_fini": [ + "out_fini" + ], + "ERR": [ + "out_err" + ], + "Print": [ + "out_print_func" + ], + "common_init": [ + "core_init", + "util_mmap_init" + ], + "common_fini": [ + "util_mmap_fini", + "core_fini" + ], + "Last_errormsg_key_alloc": [ + "_Last_errormsg_key_alloc" + ], + "_Last_errormsg_key_alloc": [ + "os_once", + "os_tls_key_create" + ], + "flush_empty": [ + "flush_empty_nolog" + ], + "pmem_drain": [ + "fence_empty", + "memory_barrier" + ], + "pmem_deep_flush": [ + "flush_empty", + "flush_clflush", + "flush_clflushopt", + "flush_clwb" + ], + "pmem_flush": [ + "flush_empty", + "flush_clflush", + "flush_clflushopt", + "flush_clwb" + ], + "pmem_memmove": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memcpy": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memmove_nodrain": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memcpy_nodrain": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memmove_persist": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memcpy_persist": [ + "memmove_nodrain_libc", + "memmove_nodrain_generic", + "pmem_memmove_nodrain.static", + "pmem_memmove_nodrain_eadr.static" + ], + "pmem_memset": [ + "memset_nodrain_libc", + "memset_nodrain_generic", + "pmem_memset_nodrain.static", + "pmem_memset_nodrain_eadr.static" + ], + "pmem_memset_nodrain": [ + "memset_nodrain_libc", + "memset_nodrain_generic", + "pmem_memset_nodrain.static", + "pmem_memset_nodrain_eadr.static" + ], + "pmem_memset_persist": [ + "memset_nodrain_libc", + "memset_nodrain_generic", + "pmem_memset_nodrain.static", + "pmem_memset_nodrain_eadr.static" + ], + "pmem_memmove_nodrain.static": [ + "memmove_mov_sse2_noflush", + "memmove_mov_avx_noflush", + "memmove_mov_avx512f_noflush", + "memmove_movnt_sse2_clflush_wcbarrier", + "memmove_movnt_sse2_clflush_nobarrier", + "memmove_movnt_sse2_clflushopt_wcbarrier", + "memmove_movnt_sse2_clflushopt_nobarrier", + "memmove_movnt_sse2_clwb_wcbarrier", + "memmove_movnt_sse2_clwb_nobarrier", + "memmove_movnt_avx_clflush_wcbarrier", + "memmove_movnt_avx_clflush_nobarrier", + "memmove_movnt_avx_clflushopt_wcbarrier", + "memmove_movnt_avx_clflushopt_nobarrier", + "memmove_movnt_avx_clwb_wcbarrier", + "memmove_movnt_avx_clwb_nobarrier", + "memmove_movnt_avx512f_clflush", + "memmove_movnt_avx512f_clflushopt", + "memmove_movnt_avx512f_clwb", + "memmove_movnt_movdir64b_clflush", + "memmove_movnt_movdir64b_clflushopt", + "memmove_movnt_movdir64b_clwb", + "memmove_mov_sse2_clflush", + "memmove_mov_sse2_clflushopt", + "memmove_mov_sse2_clwb", + "memmove_mov_avx_clflush", + "memmove_mov_avx_clflushopt", + "memmove_mov_avx_clwb", + "memmove_mov_avx512f_clflush", + "memmove_mov_avx512f_clflushopt", + "memmove_mov_avx512f_clwb" + ], + "pmem_memmove_nodrain_eadr.static": [ + "memmove_mov_sse2_noflush", + "memmove_mov_avx_noflush", + "memmove_mov_avx512f_noflush", + "memmove_movnt_sse2_empty_wcbarrier", + "memmove_movnt_sse2_empty_nobarrier", + "memmove_movnt_avx_empty_wcbarrier", + "memmove_movnt_avx_empty_nobarrier", + "memmove_movnt_avx512f_empty", + "memmove_movnt_movdir64b_empty", + "memmove_mov_sse2_empty", + "memmove_mov_avx_empty", + "memmove_mov_avx512f_empty" + ], + "pmem_memset_nodrain.static": [ + "memset_mov_sse2_noflush", + "memset_mov_avx_noflush", + "memset_mov_avx512f_noflush", + "memset_movnt_sse2_clflush_wcbarrier", + "memset_movnt_sse2_clflush_nobarrier", + "memset_movnt_sse2_clflushopt_wcbarrier", + "memset_movnt_sse2_clflushopt_nobarrier", + "memset_movnt_sse2_clwb_wcbarrier", + "memset_movnt_sse2_clwb_nobarrier", + "memset_movnt_avx_clflush_wcbarrier", + "memset_movnt_avx_clflush_nobarrier", + "memset_movnt_avx_clflushopt_wcbarrier", + "memset_movnt_avx_clflushopt_nobarrier", + "memset_movnt_avx_clwb_wcbarrier", + "memset_movnt_avx_clwb_nobarrier", + "memset_movnt_avx512f_clflush", + "memset_movnt_avx512f_clflushopt", + "memset_movnt_avx512f_clwb", + "memset_movnt_movdir64b_clflush", + "memset_movnt_movdir64b_clflushopt", + "memset_movnt_movdir64b_clwb", + "memset_mov_sse2_clflush", + "memset_mov_sse2_clflushopt", + "memset_mov_sse2_clwb", + "memset_mov_avx_clflush", + "memset_mov_avx_clflushopt", + "memset_mov_avx_clwb", + "memset_mov_avx512f_clflush", + "memset_mov_avx512f_clflushopt", + "memset_mov_avx512f_clwb" + ], + "pmem_memset_nodrain_eadr.static": [ + "memset_mov_sse2_noflush", + "memset_mov_avx_noflush", + "memset_mov_avx512f_noflush", + "memset_movnt_sse2_empty_wcbarrier", + "memset_movnt_sse2_empty_nobarrier", + "memset_movnt_avx_empty_wcbarrier", + "memset_movnt_avx_empty_nobarrier", + "memset_movnt_avx512f_empty", + "memset_movnt_movdir64b_empty", + "memset_mov_sse2_empty", + "memset_mov_avx_empty", + "memset_mov_avx512f_empty" + ] + }, + "white_list": [ + "__comment__ info->memset_funcs.nt.noflush is never used", + "memset_movnt_movdir64b_noflush", + "memset_movnt_avx512f_noflush", + "memset_movnt_avx_noflush_wcbarrier", + "memset_movnt_avx_noflush_nobarrier", + "memset_movnt_sse2_noflush_wcbarrier", + "memset_movnt_sse2_noflush_nobarrier", + "__comment__ info->memmove_funcs.nt.noflush is never used", + "memmove_movnt_avx512f_noflush", + "memmove_movnt_movdir64b_noflush", + "memmove_movnt_avx_noflush_wcbarrier", + "memmove_movnt_avx_noflush_nobarrier", + "memmove_movnt_sse2_noflush_wcbarrier", + "memmove_movnt_sse2_noflush_nobarrier", + "out_fatal", + "pmem_emit_log", + "util_emit_log", + "pmem2_flush_file_buffers_os", + "util_file_zero", + "util_file_map_whole", + "pmem2_device_dax_alignment", + "util_file_open", + "util_unlink", + "util_fgets", + "set_func_realloc", + "set_func_malloc", + "pmem2_config_init", + "os_fsync", + "os_flock", + "util_checksum_compute", + "util_header_check", + "util_poolset_parse", + "util_parse_size", + "util_compare_file_inodes", + "util_replica_open", + "util_replica_map_local.constprop", + "util_poolset_chmod", + "pmem2_extents_create_get", + "out_nonl", + "out_log", + "out_fatal", + "util_header_create", + "util_map_part", + "util_pool_open", + "util_map_hdr", + "util_fd_get_type", + "util_replica_close", + "badblocks_get", + "pmem2_badblock_context_new", + "util_poolset_create_set", + "util_poolset_append_new_part.part.2", + "shutdown_state_add_part", + "ctl_find_node.isra.5", + "badblocks_clear", + "util_uuid_to_string", + "util_uuid_from_string", + "util_replica_close_local", + "util_poolset_foreach_part_struct", + "util_pool_extend", + "ctl_query", + "ctl_load_config", + "ctl_exec_query_write", + "badblocks_recovery_file_alloc", + "badblocks_clear_all", + "util_uuid_generate", + "util_unmap_part", + "util_unmap_hdr", + "util_str2feature", + "util_poolset_single", + "util_pool_create_uuids", + "util_part_open", + "ravl_interval_find", + "ravl_find", + "_Last_errormsg_key_alloc", + "ctl_load_config_from_file", + "util_poolset_free", + "util_poolset_foreach_part", + "util_poolset_files_local", + "util_is_poolset_file", + "ravl_emplace", + "pmem2_source_numa_node", + "pmem2_source_device_usc", + "pmem2_source_device_id", + "pmem2_badblock_next", + "pmem2_badblock_clear", + "os_part_deep_common", + "ctl_arg_integer", + "util_write_all", + "util_unmap_all_hdrs", + "util_replica_reserve", + "util_replica_deep_common", + "util_replica_add_part_by_idx", + "util_readline", + "util_poolset_size", + "util_poolset_read", + "util_poolset_close", + "util_pool_open_nocheck", + "util_localtime", + "util_file_pwrite", + "util_file_pread", + "util_file_device_dax_alignment", + "util_file_create", + "util_feature_check", + "util_feature2str", + "util_feature2pmempool_feature", + "util_concat_str", + "util_aligned_malloc", + "shutdown_state_reinit.isra.1", + "ravl_interval_insert", + "ravl_interval_find_prev", + "ravl_interval_find_next", + "ravl_interval_find_equal", + "ravl_foreach_node", + "pmem2_source_from_anon", + "pmem2_badblock_next_region", + "badblocks_recovery_file_exists", + "util_unmap_parts", + "util_unlink_flock", + "util_str2pmempool_feature", + "util_set_alloc_funcs", + "util_replica_fdclose", + "util_replica_check_map_sync", + "util_poolset_set_size", + "util_poolset_open", + "util_poolset_fdclose_always", + "util_parse_add_replica", + "util_file_dir_next", + "util_checksum", + "util_check_arch_flags", + "shutdown_state_set_dirty", + "shutdown_state_clear_dirty", + "shutdown_state_checksum", + "shutdown_state_check", + "ravl_remove", + "ravl_new_sized", + "ravl_interval_new", + "ravl_interval_compare", + "pmem2_zalloc", + "pmem2_realloc", + "pmem2_extents_destroy", + "pmem2_deep_flush_dax", + "pmem2_config_new", + "pmem2_badblock_context_delete", + "os_fsync_dir", + "membuf_new", + "membuf_key_destructor", + "membuf_delete", + "membuf_alloc", + "ctl_load_config_from_string", + "ctl_arg_string", + "ctl_arg_boolean", + "badblocks_delete", + "badblocks_count", + "badblocks_clear_poolset", + "badblocks_check_poolset", + "badblocks_check_file_cb", + "util_safe_strcpy", + "util_range_rw", + "util_range_ro", + "util_range_none", + "util_print_bad_files_cb", + "util_part_fdclose", + "util_is_zeroed", + "util_file_dir_open", + "util_checksum_seq", + "shutdown_state_init", + "ravl_interval_remove", + "ravl_interval_find_last", + "ravl_interval_find_first", + "ravl_interval_delete", + "ravl_interval_delete_cb", + "ravl_delete_cb", + "ravl_clear", + "randomize_r", + "pmem2_source_get_fd", + "pmem2_source_alignment", + "pmem2_config_validate_length", + "pmem2_config_set_sharing", + "pmem2_config_set_required_store_granularity", + "pmem2_config_set_protection", + "pmem2_config_set_offset", + "pmem2_config_delete", + "pmem2_badblock_next_namespace", + "os_thread_self", + "ctl_new", + "ctl_exec_query_read", + "badblocks_new", + "badblocks_clear_poolset_cb", + "badblocks_check_file", + "util_replica_force_page_allocation", + "util_replica_deep_persist", + "util_replica_deep_drain", + "util_poolset_fdclose", + "util_pool_hdr2attr", + "util_pool_has_device_dax", + "util_pool_create", + "util_pool_attr2hdr", + "util_part_realpath", + "util_is_absolute_path", + "util_get_unknown_features", + "util_get_arch_flags", + "util_file_mkdir", + "util_file_dir_remove", + "util_file_dir_close", + "util_feature_is_zero", + "util_feature_is_set", + "util_feature_enable", + "util_feature_disable", + "util_feature_cmp", + "util_convert2le_hdr", + "util_convert2h_hdr_nocheck", + "util_aligned_free", + "rnd64", + "rnd64_r", + "_Realloc", + "ravl_node_insert_constructor", + "ravl_node_copy_constructor", + "ravl_new", + "ravl_last", + "ravl_interval_data", + "ravl_insert", + "ravl_foreach", + "ravl_first", + "ravl_empty", + "ravl_emplace_copy", + "ravl_delete", + "ravl_data", + "randomize", + "pmem_emit_log", + "pmem2_region_get_next_badblock", + "pmem2_region_get_first_badblock", + "pmem2_namespace_get_next_badblock", + "pmem2_namespace_get_first_badblock", + "pmem2_config_set_vm_reservation", + "pmem2_config_set_length", + "out_set_vsnprintf_func", + "out_set_print_func", + "os_writev", + "os_unsetenv", + "os_thread_setaffinity_np", + "os_thread_join", + "os_thread_create", + "os_thread_atfork", + "os_strsignal", + "os_spin_unlock", + "os_spin_trylock", + "os_spin_lock", + "os_spin_init", + "os_spin_destroy", + "os_setenv", + "os_semaphore_wait", + "os_semaphore_trywait", + "os_semaphore_post", + "os_semaphore_init", + "os_semaphore_destroy", + "os_rwlock_wrlock", + "os_rwlock_unlock", + "os_rwlock_trywrlock", + "os_rwlock_tryrdlock", + "os_rwlock_timedwrlock", + "os_rwlock_timedrdlock", + "os_rwlock_rdlock", + "os_rwlock_init", + "os_rwlock_destroy", + "os_rand_r", + "os_mutex_unlock", + "os_mutex_trylock", + "os_mutex_timedlock", + "os_mutex_lock", + "os_mutex_init", + "os_mutex_destroy", + "os_fdopen", + "os_execv", + "os_cpu_zero", + "os_cpu_set", + "os_cond_wait", + "os_cond_timedwait", + "os_cond_signal", + "os_cond_init", + "os_cond_destroy", + "os_cond_broadcast", + "os_clock_gettime", + "os_chmod", + "membuf_ptr_user_data", + "membuf_free", + "_Malloc", + "hash64", + "ctl_sds_register", + "ctl_register_module_node", + "ctl_prefault_register", + "ctl_fallocate_register", + "ctl_exec_query_runnable", + "ctl_delete", + "ctl_cow_register", + "ctl__at_open_write", + "ctl__at_open_read", + "ctl__at_create_write", + "ctl__at_create_read" + ] +} \ No newline at end of file diff --git a/utils/call_stacks_analysis/libpmem/make_config.py b/utils/call_stacks_analysis/libpmem/make_config.py new file mode 100755 index 00000000000..edd631fb06c --- /dev/null +++ b/utils/call_stacks_analysis/libpmem/make_config.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +import json +from typing import Dict, List + +# https://peps.python.org/pep-0613/ requires Python >= 3.10 +# from typing import TypeAlias + +# Calls: TypeAlias = dict[str, list[str]] # for Python >= 3.10 +Calls = Dict[str, List[str]] # for Python < 3.10 + +API = [ + 'pmem_map_file', + 'pmem_unmap', + 'pmem_memset', + 'pmem_memmove', + 'pmem_memcpy', + 'pmem_memset_persist', + 'pmem_memmove_persist', + 'pmem_memcpy_persist', + 'pmem_memcpy_nodrain', + "pmem_deep_persist", + "pmem_persist", + "pmem_check_version", + "libpmem_init", + "libpmem_fini", + "pmem_has_hw_drain", + "pmem_has_auto_flush", + "pmem_errormsg", + "pmem_memset_persist", + "pmem_memmove_persist" +] + +def inlines(calls: Calls) -> Calls: + calls['core_init'] = ['util_init', 'out_init'] + calls['core_fini'] = ['out_fini'] + calls['ERR'] = ['out_err'] + calls['Print'] = ['out_print_func'] + calls['common_init'] = ['core_init', 'util_mmap_init'] + calls['common_fini'] = ['util_mmap_fini', 'core_fini'] + calls['Last_errormsg_key_alloc'] = ['_Last_errormsg_key_alloc'] + calls['_Last_errormsg_key_alloc'] = ['os_once', 'os_tls_key_create'] + calls['flush_empty'] = ['flush_empty_nolog'] + return calls + +def function_pointers(calls: Calls) -> Calls: + calls['pmem_drain'] = ['fence_empty', 'memory_barrier'] + + flush_all = ['flush_empty', 'flush_clflush', 'flush_clflushopt', 'flush_clwb'] + calls['pmem_deep_flush'] = flush_all + calls['pmem_flush'] = flush_all + + # '.static' suffix added to differentiate between libpmem API function and a static helper. + memmove_nodrain_all = ['memmove_nodrain_libc', 'memmove_nodrain_generic', 'pmem_memmove_nodrain.static', 'pmem_memmove_nodrain_eadr.static'] + calls['pmem_memmove'] = memmove_nodrain_all + calls['pmem_memcpy'] = memmove_nodrain_all + calls['pmem_memmove_nodrain'] = memmove_nodrain_all + calls['pmem_memcpy_nodrain'] = memmove_nodrain_all + calls['pmem_memmove_persist'] = memmove_nodrain_all + calls['pmem_memcpy_persist'] = memmove_nodrain_all + + memset_nodrain_all = ['memset_nodrain_libc', 'memset_nodrain_generic', 'pmem_memset_nodrain.static', 'pmem_memset_nodrain_eadr.static'] + calls['pmem_memset'] = memset_nodrain_all + calls['pmem_memset_nodrain'] = memset_nodrain_all + calls['pmem_memset_persist'] = memset_nodrain_all + + memmove_funcs = { + 't': { + func: [ f'memmove_mov_{trick}_{func}' + for trick in ['sse2', 'avx', 'avx512f'] + ] for func in ['noflush', 'empty'] + }, + 'nt': { + 'empty': [ f'memmove_movnt_{trick}_empty_{drain}' + for trick in ['sse2', 'avx'] + for drain in ['wcbarrier', 'nobarrier'] + ], + 'flush': [ f'memmove_movnt_{trick}_{flush}_{drain}' + for trick in ['sse2', 'avx'] + for flush in ['clflush', 'clflushopt', 'clwb'] + for drain in ['wcbarrier', 'nobarrier'] + ] + } + } + memmove_funcs_extras = { + 't': { + 'flush': [ f'memmove_mov_{trick}_{flush}' + for trick in ['sse2', 'avx', 'avx512f'] + for flush in ['clflush', 'clflushopt', 'clwb'] + ] + }, + 'nt': { + 'empty': [ f'memmove_movnt_{trick}_empty' + for trick in ['avx512f', 'movdir64b'] + ], + 'flush': [ f'memmove_movnt_{trick}_{flush}' + for trick in ['avx512f', 'movdir64b'] + for flush in ['clflush', 'clflushopt', 'clwb'] + ] + } + } + memmove_funcs['t']['flush'] = memmove_funcs_extras['t']['flush'] + memmove_funcs['nt']['empty'].extend(memmove_funcs_extras['nt']['empty']) + memmove_funcs['nt']['flush'].extend(memmove_funcs_extras['nt']['flush']) + + calls['pmem_memmove_nodrain.static'] = \ + memmove_funcs['t']['noflush'] + \ + memmove_funcs['nt']['flush'] + \ + memmove_funcs['t']['flush'] + + calls['pmem_memmove_nodrain_eadr.static'] = \ + memmove_funcs['t']['noflush'] + \ + memmove_funcs['nt']['empty'] + \ + memmove_funcs['t']['empty'] + + memsetfuncs = { + 't': { + func: [ f'memset_mov_{trick}_{func}' + for trick in ['sse2', 'avx', 'avx512f'] + ] for func in ['noflush', 'empty'] + }, + 'nt': { + 'empty': [ f'memset_movnt_{trick}_empty_{drain}' + for trick in ['sse2', 'avx'] + for drain in ['wcbarrier', 'nobarrier'] + ], + 'flush': [ f'memset_movnt_{trick}_{flush}_{drain}' + for trick in ['sse2', 'avx'] + for flush in ['clflush', 'clflushopt', 'clwb'] + for drain in ['wcbarrier', 'nobarrier'] + ] + } + } + memsetfuncs_extras = { + 't': { + 'flush': [ f'memset_mov_{trick}_{flush}' + for trick in ['sse2', 'avx', 'avx512f'] + for flush in ['clflush', 'clflushopt', 'clwb'] + ] + }, + 'nt': { + 'empty': [ f'memset_movnt_{trick}_empty' + for trick in ['avx512f', 'movdir64b'] + ], + 'flush': [ f'memset_movnt_{trick}_{flush}' + for trick in ['avx512f', 'movdir64b'] + for flush in ['clflush', 'clflushopt', 'clwb'] + ] + } + } + memsetfuncs['t']['flush'] = memsetfuncs_extras['t']['flush'] + memsetfuncs['nt']['empty'].extend(memsetfuncs_extras['nt']['empty']) + memsetfuncs['nt']['flush'].extend(memsetfuncs_extras['nt']['flush']) + + calls['pmem_memset_nodrain.static'] = \ + memsetfuncs['t']['noflush'] + \ + memsetfuncs['nt']['flush'] + \ + memsetfuncs['t']['flush'] + + calls['pmem_memset_nodrain_eadr.static'] = \ + memsetfuncs['t']['noflush'] + \ + memsetfuncs['nt']['empty'] + \ + memsetfuncs['t']['empty'] + + return calls + +def main(): + with open("white_list.json", 'r') as file: + white_list = json.load(file) + extra_calls = inlines({}) + extra_calls = function_pointers(extra_calls) + config = { + 'filter': 'libpmem', + 'api': API, + 'dead_end': [], + 'extra_calls': extra_calls, + 'white_list': white_list + } + with open("config.json", "w") as outfile: + json.dump(config, outfile, indent = 4) + +if __name__ == '__main__': + main() diff --git a/utils/call_stacks_analysis/libpmem/white_list.json b/utils/call_stacks_analysis/libpmem/white_list.json new file mode 100644 index 00000000000..2c0621d22d0 --- /dev/null +++ b/utils/call_stacks_analysis/libpmem/white_list.json @@ -0,0 +1,304 @@ +[ + "__comment__ info->memset_funcs.nt.noflush is never used", + + "memset_movnt_movdir64b_noflush", + "memset_movnt_avx512f_noflush", + "memset_movnt_avx_noflush_wcbarrier", + "memset_movnt_avx_noflush_nobarrier", + "memset_movnt_sse2_noflush_wcbarrier", + "memset_movnt_sse2_noflush_nobarrier", + + "__comment__ info->memmove_funcs.nt.noflush is never used", + + "memmove_movnt_avx512f_noflush", + "memmove_movnt_movdir64b_noflush", + "memmove_movnt_avx_noflush_wcbarrier", + "memmove_movnt_avx_noflush_nobarrier", + "memmove_movnt_sse2_noflush_wcbarrier", + "memmove_movnt_sse2_noflush_nobarrier", + + "out_fatal", + "pmem_emit_log", + "util_emit_log", + "pmem2_flush_file_buffers_os", + "util_file_zero", + "util_file_map_whole", + "pmem2_device_dax_alignment", + "util_file_open", + "util_unlink", + "util_fgets", + "set_func_realloc", + "set_func_malloc", + "pmem2_config_init", + "os_fsync", + "os_flock", + "util_checksum_compute", + "util_header_check", + "util_poolset_parse", + "util_parse_size", + "util_compare_file_inodes", + "util_replica_open", + "util_replica_map_local.constprop", + "util_poolset_chmod", + "pmem2_extents_create_get", + "out_nonl", + "out_log", + "out_fatal", + "util_header_create", + "util_map_part", + "util_pool_open", + "util_map_hdr", + "util_fd_get_type", + "util_replica_close", + "badblocks_get", + "pmem2_badblock_context_new", + "util_poolset_create_set", + "util_poolset_append_new_part.part.2", + "shutdown_state_add_part", + "ctl_find_node.isra.5", + "badblocks_clear", + "util_uuid_to_string", + "util_uuid_from_string", + "util_replica_close_local", + "util_poolset_foreach_part_struct", + "util_pool_extend", + "ctl_query", + "ctl_load_config", + "ctl_exec_query_write", + "badblocks_recovery_file_alloc", + "badblocks_clear_all", + "util_uuid_generate", + "util_unmap_part", + "util_unmap_hdr", + "util_str2feature", + "util_poolset_single", + "util_pool_create_uuids", + "util_part_open", + "ravl_interval_find", + "ravl_find", + "_Last_errormsg_key_alloc", + "ctl_load_config_from_file", + "util_poolset_free", + "util_poolset_foreach_part", + "util_poolset_files_local", + "util_is_poolset_file", + "ravl_emplace", + "pmem2_source_numa_node", + "pmem2_source_device_usc", + "pmem2_source_device_id", + "pmem2_badblock_next", + "pmem2_badblock_clear", + "os_part_deep_common", + "ctl_arg_integer", + "util_write_all", + "util_unmap_all_hdrs", + "util_replica_reserve", + "util_replica_deep_common", + "util_replica_add_part_by_idx", + "util_readline", + "util_poolset_size", + "util_poolset_read", + "util_poolset_close", + "util_pool_open_nocheck", + "util_localtime", + "util_file_pwrite", + "util_file_pread", + "util_file_device_dax_alignment", + "util_file_create", + "util_feature_check", + "util_feature2str", + "util_feature2pmempool_feature", + "util_concat_str", + "util_aligned_malloc", + "shutdown_state_reinit.isra.1", + "ravl_interval_insert", + "ravl_interval_find_prev", + "ravl_interval_find_next", + "ravl_interval_find_equal", + "ravl_foreach_node", + "pmem2_source_from_anon", + "pmem2_badblock_next_region", + "badblocks_recovery_file_exists", + "util_unmap_parts", + "util_unlink_flock", + "util_str2pmempool_feature", + "util_set_alloc_funcs", + "util_replica_fdclose", + "util_replica_check_map_sync", + "util_poolset_set_size", + "util_poolset_open", + "util_poolset_fdclose_always", + "util_parse_add_replica", + "util_file_dir_next", + "util_checksum", + "util_check_arch_flags", + "shutdown_state_set_dirty", + "shutdown_state_clear_dirty", + "shutdown_state_checksum", + "shutdown_state_check", + "ravl_remove", + "ravl_new_sized", + "ravl_interval_new", + "ravl_interval_compare", + "pmem2_zalloc", + "pmem2_realloc", + "pmem2_extents_destroy", + "pmem2_deep_flush_dax", + "pmem2_config_new", + "pmem2_badblock_context_delete", + "os_fsync_dir", + "membuf_new", + "membuf_key_destructor", + "membuf_delete", + "membuf_alloc", + "ctl_load_config_from_string", + "ctl_arg_string", + "ctl_arg_boolean", + "badblocks_delete", + "badblocks_count", + "badblocks_clear_poolset", + "badblocks_check_poolset", + "badblocks_check_file_cb", + "util_safe_strcpy", + "util_range_rw", + "util_range_ro", + "util_range_none", + "util_print_bad_files_cb", + "util_part_fdclose", + "util_is_zeroed", + "util_file_dir_open", + "util_checksum_seq", + "shutdown_state_init", + "ravl_interval_remove", + "ravl_interval_find_last", + "ravl_interval_find_first", + "ravl_interval_delete", + "ravl_interval_delete_cb", + "ravl_delete_cb", + "ravl_clear", + "randomize_r", + "pmem2_source_get_fd", + "pmem2_source_alignment", + "pmem2_config_validate_length", + "pmem2_config_set_sharing", + "pmem2_config_set_required_store_granularity", + "pmem2_config_set_protection", + "pmem2_config_set_offset", + "pmem2_config_delete", + "pmem2_badblock_next_namespace", + "os_thread_self", + "ctl_new", + "ctl_exec_query_read", + "badblocks_new", + "badblocks_clear_poolset_cb", + "badblocks_check_file", + "util_replica_force_page_allocation", + "util_replica_deep_persist", + "util_replica_deep_drain", + "util_poolset_fdclose", + "util_pool_hdr2attr", + "util_pool_has_device_dax", + "util_pool_create", + "util_pool_attr2hdr", + "util_part_realpath", + "util_is_absolute_path", + "util_get_unknown_features", + "util_get_arch_flags", + "util_file_mkdir", + "util_file_dir_remove", + "util_file_dir_close", + "util_feature_is_zero", + "util_feature_is_set", + "util_feature_enable", + "util_feature_disable", + "util_feature_cmp", + "util_convert2le_hdr", + "util_convert2h_hdr_nocheck", + "util_aligned_free", + "rnd64", + "rnd64_r", + "_Realloc", + "ravl_node_insert_constructor", + "ravl_node_copy_constructor", + "ravl_new", + "ravl_last", + "ravl_interval_data", + "ravl_insert", + "ravl_foreach", + "ravl_first", + "ravl_empty", + "ravl_emplace_copy", + "ravl_delete", + "ravl_data", + "randomize", + "pmem_emit_log", + "pmem2_region_get_next_badblock", + "pmem2_region_get_first_badblock", + "pmem2_namespace_get_next_badblock", + "pmem2_namespace_get_first_badblock", + "pmem2_config_set_vm_reservation", + "pmem2_config_set_length", + "out_set_vsnprintf_func", + "out_set_print_func", + "os_writev", + "os_unsetenv", + "os_thread_setaffinity_np", + "os_thread_join", + "os_thread_create", + "os_thread_atfork", + "os_strsignal", + "os_spin_unlock", + "os_spin_trylock", + "os_spin_lock", + "os_spin_init", + "os_spin_destroy", + "os_setenv", + "os_semaphore_wait", + "os_semaphore_trywait", + "os_semaphore_post", + "os_semaphore_init", + "os_semaphore_destroy", + "os_rwlock_wrlock", + "os_rwlock_unlock", + "os_rwlock_trywrlock", + "os_rwlock_tryrdlock", + "os_rwlock_timedwrlock", + "os_rwlock_timedrdlock", + "os_rwlock_rdlock", + "os_rwlock_init", + "os_rwlock_destroy", + "os_rand_r", + "os_mutex_unlock", + "os_mutex_trylock", + "os_mutex_timedlock", + "os_mutex_lock", + "os_mutex_init", + "os_mutex_destroy", + "os_fdopen", + "os_execv", + "os_cpu_zero", + "os_cpu_set", + "os_cond_wait", + "os_cond_timedwait", + "os_cond_signal", + "os_cond_init", + "os_cond_destroy", + "os_cond_broadcast", + "os_clock_gettime", + "os_chmod", + "membuf_ptr_user_data", + "membuf_free", + "_Malloc", + "hash64", + "ctl_sds_register", + "ctl_register_module_node", + "ctl_prefault_register", + "ctl_fallocate_register", + "ctl_exec_query_runnable", + "ctl_delete", + "ctl_cow_register", + "ctl__at_open_write", + "ctl__at_open_read", + "ctl__at_create_write", + "ctl__at_create_read" + ] diff --git a/utils/call_stacks_analysis/libpmemobj/config.json b/utils/call_stacks_analysis/libpmemobj/config.json new file mode 100644 index 00000000000..d391fc84e48 --- /dev/null +++ b/utils/call_stacks_analysis/libpmemobj/config.json @@ -0,0 +1,645 @@ +{ + "filter": "libpmemobj", + "api": [ + "libpmemobj_init", + "pmem_init", + "pmemobj_alloc_usable_size", + "pmemobj_alloc", + "pmemobj_cancel", + "pmemobj_check_version", + "pmemobj_check", + "pmemobj_close", + "pmemobj_cond_broadcast", + "pmemobj_cond_signal", + "pmemobj_cond_wait", + "pmemobj_cond_zero", + "pmemobj_create", + "pmemobj_ctl_exec", + "pmemobj_ctl_get", + "pmemobj_ctl_set", + "pmemobj_defer_free", + "pmemobj_defrag", + "pmemobj_direct", + "pmemobj_drain", + "pmemobj_errormsg", + "pmemobj_first", + "pmemobj_flush", + "pmemobj_free", + "pmemobj_get_user_data", + "pmemobj_list_insert", + "pmemobj_list_move", + "pmemobj_list_remove", + "pmemobj_memcpy_persist", + "pmemobj_memcpy", + "pmemobj_memmove", + "pmemobj_memset_persist", + "pmemobj_memset", + "pmemobj_mutex_timedlock", + "pmemobj_mutex_trylock", + "pmemobj_mutex_zero", + "pmemobj_next", + "pmemobj_oid", + "pmemobj_open", + "pmemobj_persist", + "pmemobj_publish", + "pmemobj_realloc", + "pmemobj_reserve", + "pmemobj_root_size", + "pmemobj_root", + "pmemobj_rwlock_rdlock", + "pmemobj_rwlock_timedrdlock", + "pmemobj_rwlock_timedwrlock", + "pmemobj_rwlock_tryrdlock", + "pmemobj_rwlock_trywrlock", + "pmemobj_rwlock_zero", + "pmemobj_set_funcs", + "pmemobj_set_user_data", + "pmemobj_set_value", + "pmemobj_strdup", + "pmemobj_tx_abort", + "pmemobj_tx_add_range_direct", + "pmemobj_tx_add_range", + "pmemobj_tx_begin", + "pmemobj_tx_commit", + "pmemobj_tx_end", + "pmemobj_tx_errno", + "pmemobj_tx_free", + "pmemobj_tx_get_failure_behavior", + "pmemobj_tx_get_user_data", + "pmemobj_tx_lock", + "pmemobj_tx_log_append_buffer", + "pmemobj_tx_log_auto_alloc", + "pmemobj_tx_log_intents_max_size", + "pmemobj_tx_log_snapshots_max_size", + "pmemobj_tx_process", + "pmemobj_tx_publish", + "pmemobj_tx_realloc", + "pmemobj_tx_set_failure_behavior", + "pmemobj_tx_set_user_data", + "pmemobj_tx_stage", + "pmemobj_tx_strdup", + "pmemobj_tx_wcsdup", + "pmemobj_tx_xadd_range_direct", + "pmemobj_tx_xadd_range", + "pmemobj_tx_xalloc", + "pmemobj_tx_xlock", + "pmemobj_tx_xlog_append_buffer", + "pmemobj_tx_xstrdup", + "pmemobj_tx_xwcsdup", + "pmemobj_tx_zalloc", + "pmemobj_tx_zrealloc", + "pmemobj_type_num", + "pmemobj_volatile", + "pmemobj_xreserve", + "pmemobj_tx_alloc", + "pmemobj_wcsdup", + "pmemobj_xalloc", + "pmemobj_zalloc", + "pmemobj_zrealloc", + "pmemobj_xflush", + "pmemobj_xpersist" + ], + "dead_end": [ + "prealloc" + ], + "extra_calls": { + "core_init": [ + "util_init", + "out_init" + ], + "core_fini": [ + "out_fini" + ], + "ERR": [ + "out_err" + ], + "Print": [ + "out_print_func" + ], + "common_init": [ + "core_init", + "util_mmap_init" + ], + "common_fini": [ + "util_mmap_fini", + "core_fini" + ], + "Last_errormsg_key_alloc": [ + "_Last_errormsg_key_alloc" + ], + "_Last_errormsg_key_alloc": [ + "os_once", + "os_tls_key_create" + ], + "flush_empty": [ + "flush_empty_nolog" + ], + "libpmemobj_init": [ + "common_init" + ], + "out_common": [ + "out_snprintf" + ], + "run_vg_init": [ + "run_iterate_used" + ], + "palloc_heap_action_on_unlock": [ + "palloc_reservation_clear" + ], + "palloc_heap_action_on_cancel": [ + "palloc_reservation_clear", + "block_invalidate" + ], + "util_uuid_generate": [ + "util_uuid_from_string" + ], + "bucket_insert_block": [ + "container_ravl_insert_block", + "container_seglists_insert_block" + ], + "bucket_fini": [ + "container_ravl_destroy", + "container_seglists_destroy" + ], + "heap_free_chunk_reuse": [ + "huge_prep_operation_hdr", + "run_prep_operation_hdr" + ], + "palloc_heap_action_exec": [ + "huge_prep_operation_hdr", + "run_prep_operation_hdr" + ], + "alloc_prep_block": [ + "block_write_header" + ], + "heap_get_bestfit_block": [ + "huge_ensure_header_type", + "run_ensure_header_type" + ], + "palloc_vg_register_alloc": [ + "block_reinit_header" + ], + "heap_vg_open": [ + "huge_vg_init", + "run_vg_init" + ], + "bucket_attach_run": [ + "huge_iterate_free", + "run_iterate_free" + ], + "heap_zone_foreach_object": [ + "huge_iterate_used", + "run_iterate_used" + ], + "recycler_element_new": [ + "run_calc_free" + ], + "block_write_header": [ + "memblock_header_legacy_write", + "memblock_header_compact_write", + "memblock_header_none_write" + ], + "block_invalidate": [ + "memblock_header_legacy_invalidate", + "memblock_header_compact_invalidate", + "memblock_header_none_invalidate" + ], + "block_reinit_header": [ + "memblock_header_legacy_reinit", + "memblock_header_compact_reinit", + "memblock_header_none_reinit" + ], + "palloc_cancel": [ + "palloc_heap_action_on_cancel", + "palloc_mem_action_noop" + ], + "palloc_exec_actions": [ + "palloc_heap_action_on_process", + "palloc_mem_action_noop", + "palloc_heap_action_on_unlock", + "palloc_mem_action_noop" + ], + "ctl_exec_query_read": [ + "ctl__persistent_curr_allocated_read", + "ctl__transient_run_allocated_read", + "ctl__transient_run_active_read" + ], + "ctl_exec_query_write": [ + "ctl__arenas_assignment_type_write", + "ctl__desc_write", + "ctl__enabled_write" + ], + "ctl_query": [ + "ctl_exec_query_read", + "ctl_exec_query_write" + ], + "pmemops_xpersist": [ + "obj_rep_persist", + "obj_norep_persist" + ], + "pmemops_xflush": [ + "obj_rep_persist", + "obj_norep_persist" + ], + "pmemops_drain": [ + "obj_rep_drain", + "obj_norep_drain", + "operation_transient_drain" + ], + "pmemops_memcpy": [ + "obj_rep_memcpy", + "obj_norep_memcpy", + "operation_transient_memcpy" + ], + "pmemops_memmove": [ + "obj_rep_memmove", + "obj_norep_memmove" + ], + "pmemops_memset": [ + "obj_rep_memset", + "obj_norep_memset" + ], + "ulog_entry_apply": [ + "obj_rep_persist", + "obj_norep_persist", + "obj_rep_flush", + "obj_norep_flush", + "operation_transient_clean", + "" + ], + "obj_norep_persist": [ + "pmem_persist", + "obj_msync_nofail" + ], + "obj_rep_persist": [ + "pmem_persist", + "obj_msync_nofail" + ], + "obj_norep_flush": [ + "pmem_flush", + "obj_msync_nofail", + "pmem_memcpy", + "obj_nopmem_memcpy" + ], + "obj_rep_flush": [ + "pmem_flush", + "obj_msync_nofail", + "pmem_memcpy", + "obj_nopmem_memcpy" + ], + "obj_norep_drain": [ + "pmem_drain", + "obj_drain_empty" + ], + "obj_rep_drain": [ + "pmem_drain", + "obj_drain_empty" + ], + "obj_norep_memcpy": [ + "pmem_memcpy", + "obj_nopmem_memcpy" + ], + "obj_rep_memcpy": [ + "pmem_memcpy", + "obj_nopmem_memcpy" + ], + "obj_replicas_check_basic": [ + "pmem_memcpy", + "obj_nopmem_memcpy" + ], + "obj_norep_memmove": [ + "pmem_memmove", + "obj_nopmem_memmove" + ], + "obj_rep_memmove": [ + "pmem_memmove", + "obj_nopmem_memmove" + ], + "obj_norep_memset": [ + "pmem_memset", + "obj_nopmem_memset" + ], + "obj_rep_memset": [ + "pmem_memset", + "obj_nopmem_memset" + ] + }, + "white_list": [ + "buff_concat.constprop", + "check_domain_in_region", + "check_pool_hdr_uuids", + "check_pool_hdr", + "check_status_create", + "features_check", + "handle_sigbus_execute_mcsafe_op", + "list_fill_entry_redo_log", + "memmove_mov_avx_empty", + "memmove_mov_avx512f_empty", + "memmove_mov_sse2_empty", + "memmove_movnt_avx_clflush_nobarrier", + "memmove_movnt_avx_clflush_wcbarrier", + "memmove_movnt_avx_clflushopt_nobarrier", + "memmove_movnt_avx_clflushopt_wcbarrier", + "memmove_movnt_avx_clwb_nobarrier", + "memmove_movnt_avx_clwb_wcbarrier", + "memmove_movnt_avx_empty_nobarrier", + "memmove_movnt_avx_empty_wcbarrier", + "memmove_movnt_avx_noflush_nobarrier", + "memmove_movnt_avx_noflush_wcbarrier", + "memmove_movnt_avx512f_clflush", + "memmove_movnt_avx512f_clflushopt", + "memmove_movnt_avx512f_clwb", + "memmove_movnt_avx512f_empty", + "memmove_movnt_avx512f_noflush", + "memmove_movnt_movdir64b_clflush", + "memmove_movnt_movdir64b_clflushopt", + "memmove_movnt_movdir64b_clwb", + "memmove_movnt_movdir64b_empty", + "memmove_movnt_movdir64b_noflush", + "memmove_movnt_sse2_clflush_nobarrier", + "memmove_movnt_sse2_clflush_wcbarrier", + "memmove_movnt_sse2_clflushopt_nobarrier", + "memmove_movnt_sse2_clflushopt_wcbarrier", + "memmove_movnt_sse2_clwb_nobarrier", + "memmove_movnt_sse2_clwb_wcbarrier", + "memmove_movnt_sse2_empty_nobarrier", + "memmove_movnt_sse2_empty_wcbarrier", + "memmove_movnt_sse2_noflush_nobarrier", + "memmove_movnt_sse2_noflush_wcbarrier", + "memset_mov_avx_empty", + "memset_mov_avx512f_clflush", + "memset_mov_avx512f_clflushopt", + "memset_mov_avx512f_clwb", + "memset_mov_avx512f_empty", + "memset_mov_avx512f_noflush", + "memset_mov_sse2_empty", + "memset_movnt_avx_clflush_nobarrier", + "memset_movnt_avx_clflush_wcbarrier", + "memset_movnt_avx_clflushopt_nobarrier", + "memset_movnt_avx_clflushopt_wcbarrier", + "memset_movnt_avx_clwb_nobarrier", + "memset_movnt_avx_clwb_wcbarrier", + "memset_movnt_avx_empty_nobarrier", + "memset_movnt_avx_empty_wcbarrier", + "memset_movnt_avx_noflush_nobarrier", + "memset_movnt_avx_noflush_wcbarrier", + "memset_movnt_avx512f_clflush", + "memset_movnt_avx512f_clflushopt", + "memset_movnt_avx512f_clwb", + "memset_movnt_avx512f_empty", + "memset_movnt_avx512f_noflush", + "memset_movnt_movdir64b_clflush", + "memset_movnt_movdir64b_clflushopt", + "memset_movnt_movdir64b_clwb", + "memset_movnt_movdir64b_empty", + "memset_movnt_movdir64b_noflush", + "memset_movnt_sse2_clflush_nobarrier", + "memset_movnt_sse2_clflush_wcbarrier", + "memset_movnt_sse2_clflushopt_nobarrier", + "memset_movnt_sse2_clflushopt_wcbarrier", + "memset_movnt_sse2_clwb_nobarrier", + "memset_movnt_sse2_clwb_wcbarrier", + "memset_movnt_sse2_empty_nobarrier", + "memset_movnt_sse2_empty_wcbarrier", + "memset_movnt_sse2_noflush_nobarrier", + "memset_movnt_sse2_noflush_wcbarrier", + "replica_check_store_size", + "pool_params_parse.isra.2.constprop", + "pool_hdr_default_fix", + "pool_hdr_default_check", + "pool_set_type", + "list_insert_new", + "util_emit_log", + "palloc_defrag", + "replica_sync", + "pool_params_parse", + "util_tmpfile_mkstemp", + "util_compare_file_inodes", + "replica_check_poolset_health", + "replica_transform", + "pmem2_flush_file_buffers_os", + "pmem2_map_new", + "list_move", + "list_remove_free_user", + "out_nonl", + "out_log", + "out_fatal", + "buff_concat", + "pool_set_part_copy", + "pmem2_perror", + "replica_check_local_part_dir", + "pool_data_alloc", + "pmem_map_file", + "pmem2_vm_reservation_map_find", + "list_insert", + "rm_local", + "pool_copy", + "pmempool_rm", + "pmem2_map_delete", + "pmem2_auto_flush", + "vm_reservation_map_find_acquire", + "tx_realloc_common", + "pmemobj_list_insert_new", + "pmemobj_cond_timedwait", + "pmem2_map_find", + "palloc_next", + "palloc_first", + "obj_realloc_common", + "util_fd_get_type", + "sds_check_replica", + "pool_hdr_uuid_links", + "pool_hdr_uuid_links", + "pmem2_persist_pages", + "list_remove", + "pmem2_map_from_existing", + "pmalloc", + "memset_mov_avx_noflush", + "memset_mov_avx_clwb", + "memset_mov_avx_clflush", + "memset_mov_avx_clflushopt", + "list_insert_new_user", + "poolset_open", + "pmempool_check", + "pmempool_check_end", + "pmempool_check_init", + "pmempool_check_version", + "pmempool_errormsg", + "pmempool_feature_disable", + "pmempool_feature_enable", + "pmempool_feature_query", + "pmempool_sync", + "pmempool_transform", + "pmem_check_version", + "pmem_deep_drain", + "pmem_deep_persist", + "pmem_emit_log", + "pmem_errormsg", + "pmem_has_auto_flush", + "pmem_has_hw_drain", + "pmem_map_register", + "pmem_memcpy_nodrain", + "pmem_memcpy_persist", + "pmem_memmove_nodrain", + "pmem_memmove_nodrain_eadr", + "pmem_memmove_persist", + "pmem_memset_nodrain", + "pmem_memset_nodrain_eadr", + "pmem_memset_persist", + "pmem_os_init", + "pmem_unmap", + "check_answer_loop", + "check_backup", + "check_bad_blocks", + "check_clear_status_cache", + "check_data_alloc", + "check_data_free", + "check_end", + "check_fini", + "check_get_step_data", + "check_get_time_str", + "check_get_uuid_str", + "check_has_answer", + "check_has_error", + "check_init", + "check_is_end", + "check_is_end_util", + "check_pop_error", + "check_pop_info", + "check_pop_question", + "check_push_answer", + "check_questions_sequence_validate", + "check_sds", + "check_status_get", + "check_status_get_util", + "check_status_release", + "check_step", + "check_step_get", + "check_step_inc", + "is_cpu_avx_present", + "is_cpu_avx512f_present", + "is_cpu_clflush_present", + "is_cpu_clflushopt_present", + "is_cpu_clwb_present", + "is_cpu_genuine_intel", + "is_cpu_movdir64b_present", + "memmove_mov_avx_clflush", + "memmove_mov_avx_clflushopt", + "memmove_mov_avx_clwb", + "memmove_mov_avx_noflush", + "memmove_mov_avx512f_clflush", + "memmove_mov_avx512f_clflushopt", + "memmove_mov_avx512f_clwb", + "memmove_mov_avx512f_noflush", + "memmove_mov_sse2_clflush", + "memmove_mov_sse2_clflushopt", + "memmove_mov_sse2_clwb", + "memmove_mov_sse2_noflush", + "memmove_nodrain_libc", + "memset_mov_sse2_clflush", + "memset_mov_sse2_clflushopt", + "memset_mov_sse2_clwb", + "memset_mov_sse2_noflush", + "memset_nodrain_generic", + "memset_nodrain_libc", + "memmove_nodrain_generic", + "pmem2_arch_init", + "pmem2_badblock_clear", + "pmem2_deep_flush_byte", + "pmem2_deep_flush_cache", + "pmem2_flush_cpu_cache", + "pmem2_memmove_eadr", + "pmem2_memmove_nonpmem", + "pmem2_memmove", + "pmem2_memset_eadr", + "pmem2_memset_nonpmem", + "pmem2_memset", + "pmem2_source_check_op_size", + "pmem2_source_from_anon", + "pmem2_source_numa_node", + "pmem2_source_pread_mcsafe", + "pmem2_source_pwrite_mcsafe", + "pmem2_vm_reservation_delete", + "pmem2_vm_reservation_extend", + "pmem2_vm_reservation_new", + "pmem2_vm_reservation_shrink", + "vm_reservation_reserve_memory", + "_get_value", + "backup_poolset_requirements", + "backup_poolset", + "badblocks_clear_all", + "badblocks_clear", + "copy_replica_data_fw", + "devdax_read", + "devdax_write", + "disable_checksum_2k", + "enable_shutdown_state", + "feature_set", + "get_min_granularity", + "get_replica_uuid", + "ctl_arg_integer", + "heap_arena_create", + "heap_force_recycle", + "heap_foreach_object", + "heap_get_thread_arena_id", + "heap_set_arena_auto", + "heap_set_narenas_max", + "list_insert_user", + "list_remove_single", + "mcsafe_op_reg_read", + "mcsafe_op_reg_write", + "os_range_deep_common", + "palloc_flags", + "palloc_usable_size", + "pmemobj_mutex_assert_locked", + "pool_hdr_nondefault_fix", + "pool_hdr_poolset_uuid_find", + "pool_hdr_preliminary_check", + "pool_hdr_uuid_find", + "pool_params_from_header", + "pool_set_file_map_headers", + "pool_set_file_unmap_headers", + "pool_set_parse", + "poolset_close", + "query_feature", + "ravl_interval_find_equal", + "ravl_interval_find_next", + "ravl_interval_find_prev", + "ravl_interval_find", + "ravl_interval_insert", + "replica_check_part_dirs", + "replica_create_poolset_health_status", + "replica_free_poolset_health_status", + "replica_is_replica_broken", + "replica_open_replica_part_files", + "replica_read_features", + "replica_remove_all_recovery_files", + "replica_remove_part", + "require_other_feature_is", + "operation_user_buffer_verify_align", + "palloc_extra", + "run_fill_pct", + "run_get_state", + "sds_check", + "stats_enabled_parser", + "status_push", + "step_exe", + "util_aligned_malloc", + "util_concat_str", + "util_feature2pmempool_feature", + "util_feature2str", + "util_file_map_whole", + "util_file_pread", + "util_file_pwrite", + "util_is_poolset_file", + "util_localtime", + "util_pool_open_nocheck", + "util_poolset_foreach_part", + "util_poolset_read", + "util_poolset_size", + "util_range_register", + "util_range_unregister", + "util_str2feature", + "util_uuid_to_string", + "util_write_all" + ] +} \ No newline at end of file diff --git a/utils/call_stacks_analysis/libpmemobj/make_config.py b/utils/call_stacks_analysis/libpmemobj/make_config.py new file mode 100755 index 00000000000..976dd0df7b4 --- /dev/null +++ b/utils/call_stacks_analysis/libpmemobj/make_config.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +import json +from typing import Dict, List + +# https://peps.python.org/pep-0613/ requires Python >= 3.10 +# from typing import TypeAlias + +# Calls: TypeAlias = dict[str, list[str]] # for Python >= 3.10 +Calls = Dict[str, List[str]] # for Python < 3.10 + +API = [ + 'libpmemobj_init', + 'pmem_init', + 'pmemobj_alloc_usable_size', + 'pmemobj_alloc', + 'pmemobj_cancel', + 'pmemobj_check_version', + 'pmemobj_check', + 'pmemobj_close', + 'pmemobj_cond_broadcast', + 'pmemobj_cond_signal', + 'pmemobj_cond_wait', + 'pmemobj_cond_zero', + 'pmemobj_create', + 'pmemobj_ctl_exec', + 'pmemobj_ctl_get', + 'pmemobj_ctl_set', + 'pmemobj_defer_free', + 'pmemobj_defrag', + 'pmemobj_direct', + 'pmemobj_drain', + 'pmemobj_errormsg', + 'pmemobj_first', + 'pmemobj_flush', + 'pmemobj_free', + 'pmemobj_get_user_data', + 'pmemobj_list_insert', + 'pmemobj_list_move', + 'pmemobj_list_remove', + 'pmemobj_memcpy_persist', + 'pmemobj_memcpy', + 'pmemobj_memmove', + 'pmemobj_memset_persist', + 'pmemobj_memset', + 'pmemobj_mutex_timedlock', + 'pmemobj_mutex_trylock', + 'pmemobj_mutex_zero', + 'pmemobj_next', + 'pmemobj_oid', + 'pmemobj_open', + 'pmemobj_persist', + 'pmemobj_publish', + 'pmemobj_realloc', + 'pmemobj_reserve', + 'pmemobj_root_size', + 'pmemobj_root', + 'pmemobj_rwlock_rdlock', + 'pmemobj_rwlock_timedrdlock', + 'pmemobj_rwlock_timedwrlock', + 'pmemobj_rwlock_tryrdlock', + 'pmemobj_rwlock_trywrlock', + 'pmemobj_rwlock_zero', + 'pmemobj_set_funcs', + 'pmemobj_set_user_data', + 'pmemobj_set_value', + 'pmemobj_strdup', + 'pmemobj_tx_abort', + 'pmemobj_tx_add_range_direct', + 'pmemobj_tx_add_range', + 'pmemobj_tx_begin', + 'pmemobj_tx_commit', + 'pmemobj_tx_end', + 'pmemobj_tx_errno', + 'pmemobj_tx_free', + 'pmemobj_tx_get_failure_behavior', + 'pmemobj_tx_get_user_data', + 'pmemobj_tx_lock', + 'pmemobj_tx_log_append_buffer', + 'pmemobj_tx_log_auto_alloc', + 'pmemobj_tx_log_intents_max_size', + 'pmemobj_tx_log_snapshots_max_size', + 'pmemobj_tx_process', + 'pmemobj_tx_publish', + 'pmemobj_tx_realloc', + 'pmemobj_tx_set_failure_behavior', + 'pmemobj_tx_set_user_data', + 'pmemobj_tx_stage', + 'pmemobj_tx_strdup', + 'pmemobj_tx_wcsdup', + 'pmemobj_tx_xadd_range_direct', + 'pmemobj_tx_xadd_range', + 'pmemobj_tx_xalloc', + 'pmemobj_tx_xlock', + 'pmemobj_tx_xlog_append_buffer', + 'pmemobj_tx_xstrdup', + 'pmemobj_tx_xwcsdup', + 'pmemobj_tx_zalloc', + 'pmemobj_tx_zrealloc', + 'pmemobj_type_num', + 'pmemobj_volatile', + 'pmemobj_xreserve', + 'pmemobj_tx_alloc', + 'pmemobj_wcsdup', + 'pmemobj_xalloc', + 'pmemobj_zalloc', + 'pmemobj_zrealloc', + "pmemobj_xflush", + "pmemobj_xpersist", +] + +DEAD_END = [ + 'prealloc' +] + +def dict_extend(dict_, key, values): + if key not in dict_.keys(): + dict_[key] = values + else: + dict_[key].extend(values) + return dict_ + +def inlines(calls): + calls['core_init'] = ['util_init', 'out_init'] + calls['core_fini'] = ['out_fini'] + calls['ERR'] = ['out_err'] + calls['Print'] = ['out_print_func'] + calls['common_init'] = ['core_init', 'util_mmap_init'] + calls['common_fini'] = ['util_mmap_fini', 'core_fini'] + calls['Last_errormsg_key_alloc'] = ['_Last_errormsg_key_alloc'] + calls['_Last_errormsg_key_alloc'] = ['os_once', 'os_tls_key_create'] + calls['flush_empty'] = ['flush_empty_nolog'] + + calls = dict_extend(calls, 'libpmemobj_init', ['common_init']) + calls = dict_extend(calls, 'out_common', ['out_snprintf']) + calls = dict_extend(calls, 'run_vg_init', ['run_iterate_used']) + + calls = dict_extend(calls, 'palloc_heap_action_on_unlock', ['palloc_reservation_clear']) + calls = dict_extend(calls, 'palloc_heap_action_on_cancel', ['palloc_reservation_clear']) + + calls = dict_extend(calls, 'util_uuid_generate', ['util_uuid_from_string']) + + return calls + +def function_pointers(calls): + # block_container_ops + insert_all = ['container_ravl_insert_block', 'container_seglists_insert_block'] + get_rm_exact_all = ['container_ravl_get_rm_block_exact'] + get_rm_bestfit_all = ['container_ravl_get_rm_block_bestfit', 'container_seglists_get_rm_block_bestfit'] + is_empty_all = ['container_ravl_is_empty', 'container_seglists_is_empty'] + rm_all_all = ['container_ravl_rm_all', 'container_seglists_rm_all'] + destroy_all = ['container_ravl_destroy', 'container_seglists_destroy'] + + calls = dict_extend(calls, 'bucket_insert_block', insert_all) + + calls = dict_extend(calls, 'bucket_fini', destroy_all) + + # memory_block_ops + block_size_all = ['huge_block_size', 'run_block_size'] + prep_hdr_all = ['huge_prep_operation_hdr', 'run_prep_operation_hdr'] + get_lock_all = ['huge_get_lock', 'run_get_lock'] + get_state_all = ['huge_get_state', 'run_get_state'] + get_user_data_all = ['block_get_user_data'] + get_real_data_all = ['huge_get_real_data', 'run_get_real_data'] + get_user_size_all = ['block_get_user_size'] + get_real_size_all = ['block_get_real_size'] + write_header_all = ['block_write_header'] + invalidate_all = ['block_invalidate'] + ensure_header_type_all = ['huge_ensure_header_type', 'run_ensure_header_type'] + reinit_header_all = ['block_reinit_header'] + vg_init_all = ['huge_vg_init', 'run_vg_init'] + get_extra_all = ['block_get_extra'] + get_flags_all = ['block_get_flags'] + iterate_free_all = ['huge_iterate_free', 'run_iterate_free'] + iterate_used_all = ['huge_iterate_used', 'run_iterate_used'] + reinit_chunk_all = ['huge_reinit_chunk', 'run_reinit_chunk'] + calc_free_all = ['run_calc_free'] + get_bitmap_all = ['run_get_bitmap'] + fill_pct_all = ['huge_fill_pct', 'run_fill_pct'] + + calls = dict_extend(calls, 'heap_free_chunk_reuse', prep_hdr_all) + calls = dict_extend(calls, 'palloc_heap_action_exec', prep_hdr_all) + + calls = dict_extend(calls, 'alloc_prep_block', write_header_all) + + calls = dict_extend(calls, 'palloc_heap_action_on_cancel', invalidate_all) + + calls = dict_extend(calls, 'heap_get_bestfit_block', ensure_header_type_all) + + calls = dict_extend(calls, 'palloc_vg_register_alloc', reinit_header_all) + + calls = dict_extend(calls, 'heap_vg_open', vg_init_all) + + calls = dict_extend(calls, 'bucket_attach_run', iterate_free_all) + + calls = dict_extend(calls, 'heap_zone_foreach_object', iterate_used_all) + + calls = dict_extend(calls, 'recycler_element_new', calc_free_all) + + # memblock_header_ops + get_size_all = ['memblock_header_legacy_get_size', 'memblock_header_compact_get_size', 'memblock_header_none_get_size'] + get_extra_all = ['memblock_header_legacy_get_extra', 'memblock_header_compact_get_extra', 'memblock_header_none_get_extra'] + get_flags_all = ['memblock_header_legacy_get_flags', 'memblock_header_compact_get_flags', 'memblock_header_none_get_flags'] + write_all = ['memblock_header_legacy_write', 'memblock_header_compact_write', 'memblock_header_none_write'] + invalidate_all = ['memblock_header_legacy_invalidate', 'memblock_header_compact_invalidate', 'memblock_header_none_invalidate'] + reinit_all = ['memblock_header_legacy_reinit', 'memblock_header_compact_reinit', 'memblock_header_none_reinit'] + + calls = dict_extend(calls, 'block_write_header', write_all) + calls = dict_extend(calls, 'block_invalidate', invalidate_all) + calls = dict_extend(calls, 'block_reinit_header', reinit_all) + + # action_funcs + exec_all = ['palloc_heap_action_exec', 'palloc_mem_action_exec'] + on_cancel_all = ['palloc_heap_action_on_cancel', 'palloc_mem_action_noop'] + on_process_all = ['palloc_heap_action_on_process', 'palloc_mem_action_noop'] + on_unlock_all = ['palloc_heap_action_on_unlock', 'palloc_mem_action_noop'] + + calls = dict_extend(calls, 'palloc_cancel', on_cancel_all) + + calls = dict_extend(calls, 'palloc_exec_actions', on_process_all + on_unlock_all) + + # DAOS used CTLs + # pmemobj_ctl_get("stats.heap.curr_allocated") + # pmemobj_ctl_get("stats.heap.run_allocated") + # pmemobj_ctl_get("stats.heap.run_active") + get_all = ['ctl__persistent_curr_allocated_read', 'ctl__transient_run_allocated_read', 'ctl__transient_run_active_read'] + calls = dict_extend(calls, 'ctl_exec_query_read', get_all) + + # pmemobj_ctl_set("heap.arenas_assignment_type") + # pmemobj_ctl_set("heap.alloc_class.new.desc", &pmemslab); + # pmemobj_ctl_set("stats.enabled", &enabled); + # pmemobj_ctl_set("stats.enabled", &enabled); + set_all = ['ctl__arenas_assignment_type_write', 'ctl__desc_write', 'ctl__enabled_write'] + calls = dict_extend(calls, 'ctl_exec_query_write', set_all) + + calls = dict_extend(calls, 'ctl_query', ['ctl_exec_query_read', 'ctl_exec_query_write']) + + # pmem_ops + persist_all = ['obj_rep_persist', 'obj_norep_persist'] + flush_all = ['obj_rep_flush', 'obj_norep_flush', 'operation_transient_clean', ''] + drain_all = ['obj_rep_drain', 'obj_norep_drain', 'operation_transient_drain'] + memcpy_all = ['obj_rep_memcpy', 'obj_norep_memcpy', 'operation_transient_memcpy'] + memmove_all = ['obj_rep_memmove', 'obj_norep_memmove'] + memset_all = ['obj_rep_memset', 'obj_norep_memset'] + + calls = dict_extend(calls, 'pmemops_xpersist', persist_all) + calls = dict_extend(calls, 'pmemops_xflush', persist_all) + calls = dict_extend(calls, 'pmemops_drain', drain_all) + calls = dict_extend(calls, 'pmemops_memcpy', memcpy_all) + calls = dict_extend(calls, 'pmemops_memmove', memmove_all) + calls = dict_extend(calls, 'pmemops_memset', memset_all) + calls = dict_extend(calls, 'ulog_entry_apply', persist_all + flush_all) + + # per-replica functions + persist_local_all = ['pmem_persist', 'obj_msync_nofail'] + flush_local_all = ['pmem_flush', 'obj_msync_nofail'] + drain_local_all = ['pmem_drain', 'obj_drain_empty'] + memcpy_local_all = ['pmem_memcpy', 'obj_nopmem_memcpy'] + memmove_local_all = ['pmem_memmove', 'obj_nopmem_memmove'] + memset_local_all = ['pmem_memset', 'obj_nopmem_memset'] + + calls = dict_extend(calls, 'obj_norep_persist', persist_local_all) + calls = dict_extend(calls, 'obj_rep_persist', persist_local_all) + + calls = dict_extend(calls, 'obj_norep_flush', flush_local_all) + calls = dict_extend(calls, 'obj_rep_flush', flush_local_all) + + calls = dict_extend(calls, 'obj_norep_drain', drain_local_all) + calls = dict_extend(calls, 'obj_rep_drain', drain_local_all) + + calls = dict_extend(calls, 'obj_norep_memcpy', memcpy_local_all) + calls = dict_extend(calls, 'obj_rep_memcpy', memcpy_local_all) + calls = dict_extend(calls, 'obj_rep_flush', memcpy_local_all) + calls = dict_extend(calls, 'obj_replicas_check_basic', memcpy_local_all) + + calls = dict_extend(calls, 'obj_norep_memmove', memmove_local_all) + calls = dict_extend(calls, 'obj_rep_memmove', memmove_local_all) + + calls = dict_extend(calls, 'obj_norep_memset', memset_local_all) + calls = dict_extend(calls, 'obj_rep_memset', memset_local_all) + + return calls + +def main(): + with open("white_list.json", 'r') as file: + white_list = json.load(file) + extra_calls = inlines({}) + extra_calls = function_pointers(extra_calls) + config = { + 'filter': 'libpmemobj', + 'api': API, + 'dead_end': DEAD_END, + 'extra_calls': extra_calls, + 'white_list': white_list + } + with open("config.json", "w") as outfile: + json.dump(config, outfile, indent = 4) + +if __name__ == '__main__': + main() diff --git a/utils/call_stacks_analysis/libpmemobj/white_list.json b/utils/call_stacks_analysis/libpmemobj/white_list.json new file mode 100644 index 00000000000..3f847a46552 --- /dev/null +++ b/utils/call_stacks_analysis/libpmemobj/white_list.json @@ -0,0 +1,318 @@ +[ + "buff_concat.constprop", + "check_domain_in_region", + "check_pool_hdr_uuids", + "check_pool_hdr", + "check_status_create", + "features_check", + "handle_sigbus_execute_mcsafe_op", + "list_fill_entry_redo_log", + "memmove_mov_avx_empty", + "memmove_mov_avx512f_empty", + "memmove_mov_sse2_empty", + "memmove_movnt_avx_clflush_nobarrier", + "memmove_movnt_avx_clflush_wcbarrier", + "memmove_movnt_avx_clflushopt_nobarrier", + "memmove_movnt_avx_clflushopt_wcbarrier", + "memmove_movnt_avx_clwb_nobarrier", + "memmove_movnt_avx_clwb_wcbarrier", + "memmove_movnt_avx_empty_nobarrier", + "memmove_movnt_avx_empty_wcbarrier", + "memmove_movnt_avx_noflush_nobarrier", + "memmove_movnt_avx_noflush_wcbarrier", + "memmove_movnt_avx512f_clflush", + "memmove_movnt_avx512f_clflushopt", + "memmove_movnt_avx512f_clwb", + "memmove_movnt_avx512f_empty", + "memmove_movnt_avx512f_noflush", + "memmove_movnt_movdir64b_clflush", + "memmove_movnt_movdir64b_clflushopt", + "memmove_movnt_movdir64b_clwb", + "memmove_movnt_movdir64b_empty", + "memmove_movnt_movdir64b_noflush", + "memmove_movnt_sse2_clflush_nobarrier", + "memmove_movnt_sse2_clflush_wcbarrier", + "memmove_movnt_sse2_clflushopt_nobarrier", + "memmove_movnt_sse2_clflushopt_wcbarrier", + "memmove_movnt_sse2_clwb_nobarrier", + "memmove_movnt_sse2_clwb_wcbarrier", + "memmove_movnt_sse2_empty_nobarrier", + "memmove_movnt_sse2_empty_wcbarrier", + "memmove_movnt_sse2_noflush_nobarrier", + "memmove_movnt_sse2_noflush_wcbarrier", + "memset_mov_avx_empty", + "memset_mov_avx512f_clflush", + "memset_mov_avx512f_clflushopt", + "memset_mov_avx512f_clwb", + "memset_mov_avx512f_empty", + "memset_mov_avx512f_noflush", + "memset_mov_sse2_empty", + "memset_movnt_avx_clflush_nobarrier", + "memset_movnt_avx_clflush_wcbarrier", + "memset_movnt_avx_clflushopt_nobarrier", + "memset_movnt_avx_clflushopt_wcbarrier", + "memset_movnt_avx_clwb_nobarrier", + "memset_movnt_avx_clwb_wcbarrier", + "memset_movnt_avx_empty_nobarrier", + "memset_movnt_avx_empty_wcbarrier", + "memset_movnt_avx_noflush_nobarrier", + "memset_movnt_avx_noflush_wcbarrier", + "memset_movnt_avx512f_clflush", + "memset_movnt_avx512f_clflushopt", + "memset_movnt_avx512f_clwb", + "memset_movnt_avx512f_empty", + "memset_movnt_avx512f_noflush", + "memset_movnt_movdir64b_clflush", + "memset_movnt_movdir64b_clflushopt", + "memset_movnt_movdir64b_clwb", + "memset_movnt_movdir64b_empty", + "memset_movnt_movdir64b_noflush", + "memset_movnt_sse2_clflush_nobarrier", + "memset_movnt_sse2_clflush_wcbarrier", + "memset_movnt_sse2_clflushopt_nobarrier", + "memset_movnt_sse2_clflushopt_wcbarrier", + "memset_movnt_sse2_clwb_nobarrier", + "memset_movnt_sse2_clwb_wcbarrier", + "memset_movnt_sse2_empty_nobarrier", + "memset_movnt_sse2_empty_wcbarrier", + "memset_movnt_sse2_noflush_nobarrier", + "memset_movnt_sse2_noflush_wcbarrier", + "replica_check_store_size", + "pool_params_parse.isra.2.constprop", + "pool_hdr_default_fix", + "pool_hdr_default_check", + "pool_set_type", + "list_insert_new", + "util_emit_log", + "palloc_defrag", + "replica_sync", + "pool_params_parse", + "util_tmpfile_mkstemp", + "util_compare_file_inodes", + "replica_check_poolset_health", + "replica_transform", + "pmem2_flush_file_buffers_os", + "pmem2_map_new", + "list_move", + "list_remove_free_user", + "out_nonl", + "out_log", + "out_fatal", + "buff_concat", + "pool_set_part_copy", + "pmem2_perror", + "replica_check_local_part_dir", + "pool_data_alloc", + "pmem_map_file", + "pmem2_vm_reservation_map_find", + "list_insert", + "rm_local", + "pool_copy", + "pmempool_rm", + "pmem2_map_delete", + "pmem2_auto_flush", + "vm_reservation_map_find_acquire", + "tx_realloc_common", + "pmemobj_list_insert_new", + "pmemobj_cond_timedwait", + "pmem2_map_find", + "palloc_next", + "palloc_first", + "obj_realloc_common", + "util_fd_get_type", + "sds_check_replica", + "pool_hdr_uuid_links", + "pool_hdr_uuid_links", + "pmem2_persist_pages", + "list_remove", + "pmem2_map_from_existing", + "pmalloc", + "memset_mov_avx_noflush", + "memset_mov_avx_clwb", + "memset_mov_avx_clflush", + "memset_mov_avx_clflushopt", + "list_insert_new_user", + "poolset_open", + "pmempool_check", + "pmempool_check_end", + "pmempool_check_init", + "pmempool_check_version", + "pmempool_errormsg", + "pmempool_feature_disable", + "pmempool_feature_enable", + "pmempool_feature_query", + "pmempool_sync", + "pmempool_transform", + "pmem_check_version", + "pmem_deep_drain", + "pmem_deep_persist", + "pmem_emit_log", + "pmem_errormsg", + "pmem_has_auto_flush", + "pmem_has_hw_drain", + "pmem_map_register", + "pmem_memcpy_nodrain", + "pmem_memcpy_persist", + "pmem_memmove_nodrain", + "pmem_memmove_nodrain_eadr", + "pmem_memmove_persist", + "pmem_memset_nodrain", + "pmem_memset_nodrain_eadr", + "pmem_memset_persist", + "pmem_os_init", + "pmem_unmap", + "check_answer_loop", + "check_backup", + "check_bad_blocks", + "check_clear_status_cache", + "check_data_alloc", + "check_data_free", + "check_end", + "check_fini", + "check_get_step_data", + "check_get_time_str", + "check_get_uuid_str", + "check_has_answer", + "check_has_error", + "check_init", + "check_is_end", + "check_is_end_util", + "check_pop_error", + "check_pop_info", + "check_pop_question", + "check_push_answer", + "check_questions_sequence_validate", + "check_sds", + "check_status_get", + "check_status_get_util", + "check_status_release", + "check_step", + "check_step_get", + "check_step_inc", + "is_cpu_avx_present", + "is_cpu_avx512f_present", + "is_cpu_clflush_present", + "is_cpu_clflushopt_present", + "is_cpu_clwb_present", + "is_cpu_genuine_intel", + "is_cpu_movdir64b_present", + "memmove_mov_avx_clflush", + "memmove_mov_avx_clflushopt", + "memmove_mov_avx_clwb", + "memmove_mov_avx_noflush", + "memmove_mov_avx512f_clflush", + "memmove_mov_avx512f_clflushopt", + "memmove_mov_avx512f_clwb", + "memmove_mov_avx512f_noflush", + "memmove_mov_sse2_clflush", + "memmove_mov_sse2_clflushopt", + "memmove_mov_sse2_clwb", + "memmove_mov_sse2_noflush", + "memmove_nodrain_libc", + "memset_mov_sse2_clflush", + "memset_mov_sse2_clflushopt", + "memset_mov_sse2_clwb", + "memset_mov_sse2_noflush", + "memset_nodrain_generic", + "memset_nodrain_libc", + "memmove_nodrain_generic", + "pmem2_arch_init", + "pmem2_badblock_clear", + "pmem2_deep_flush_byte", + "pmem2_deep_flush_cache", + "pmem2_flush_cpu_cache", + "pmem2_memmove_eadr", + "pmem2_memmove_nonpmem", + "pmem2_memmove", + "pmem2_memset_eadr", + "pmem2_memset_nonpmem", + "pmem2_memset", + "pmem2_source_check_op_size", + "pmem2_source_from_anon", + "pmem2_source_numa_node", + "pmem2_source_pread_mcsafe", + "pmem2_source_pwrite_mcsafe", + "pmem2_vm_reservation_delete", + "pmem2_vm_reservation_extend", + "pmem2_vm_reservation_new", + "pmem2_vm_reservation_shrink", + "vm_reservation_reserve_memory", + "_get_value", + "backup_poolset_requirements", + "backup_poolset", + "badblocks_clear_all", + "badblocks_clear", + "copy_replica_data_fw", + "devdax_read", + "devdax_write", + "disable_checksum_2k", + "enable_shutdown_state", + "feature_set", + "get_min_granularity", + "get_replica_uuid", + "ctl_arg_integer", + "heap_arena_create", + "heap_force_recycle", + "heap_foreach_object", + "heap_get_thread_arena_id", + "heap_set_arena_auto", + "heap_set_narenas_max", + "list_insert_user", + "list_remove_single", + "mcsafe_op_reg_read", + "mcsafe_op_reg_write", + "os_range_deep_common", + "palloc_flags", + "palloc_usable_size", + "pmemobj_mutex_assert_locked", + "pool_hdr_nondefault_fix", + "pool_hdr_poolset_uuid_find", + "pool_hdr_preliminary_check", + "pool_hdr_uuid_find", + "pool_params_from_header", + "pool_set_file_map_headers", + "pool_set_file_unmap_headers", + "pool_set_parse", + "poolset_close", + "query_feature", + "ravl_interval_find_equal", + "ravl_interval_find_next", + "ravl_interval_find_prev", + "ravl_interval_find", + "ravl_interval_insert", + "replica_check_part_dirs", + "replica_create_poolset_health_status", + "replica_free_poolset_health_status", + "replica_is_replica_broken", + "replica_open_replica_part_files", + "replica_read_features", + "replica_remove_all_recovery_files", + "replica_remove_part", + "require_other_feature_is", + "operation_user_buffer_verify_align", + "palloc_extra", + "run_fill_pct", + "run_get_state", + "sds_check", + "stats_enabled_parser", + "status_push", + "step_exe", + "util_aligned_malloc", + "util_concat_str", + "util_feature2pmempool_feature", + "util_feature2str", + "util_file_map_whole", + "util_file_pread", + "util_file_pwrite", + "util_is_poolset_file", + "util_localtime", + "util_pool_open_nocheck", + "util_poolset_foreach_part", + "util_poolset_read", + "util_poolset_size", + "util_range_register", + "util_range_unregister", + "util_str2feature", + "util_uuid_to_string", + "util_write_all" +] \ No newline at end of file diff --git a/utils/call_stacks_analysis/stack_usage.py b/utils/call_stacks_analysis/stack_usage.py new file mode 100755 index 00000000000..5d395f25b52 --- /dev/null +++ b/utils/call_stacks_analysis/stack_usage.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +import argparse +import json + +from typing import Dict, Any + +# https://peps.python.org/pep-0589/ requires Python >= 3.8 +# from typing import TypedDict + +# Print out stack usage per function call for a given call stack. + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('-s', '--stack-usage-file', required=True) +PARSER.add_argument('-c', '--call-stack', required=True) + +# class StackUsageRecord(TypedDict): # for Python >= 3.8 +# size: int +StackUsageRecord = Dict[str, int] # for Python < 3.8 + +# class CallStack(TypedDict): # for Python >= 3.8 +# stack: list[str] +# size: int +CallStack = Dict[str, Any] # for Python < 3.8 + +def load_stack_usage(stack_usage_file: str) -> Dict[str, StackUsageRecord]: + with open(stack_usage_file, 'r') as file: + return json.load(file) + +def load_call_stack(call_stack: str) -> CallStack: + with open(call_stack, 'r') as file: + return json.load(file) + +def main(): + args = PARSER.parse_args() + stack_usage = load_stack_usage(args.stack_usage_file) + call_stack = load_call_stack(args.call_stack) + for func in call_stack['stack']: + if func in stack_usage.keys(): + size = stack_usage[func]['size'] + else: + size = 0 + print(f"{size}\t{func}") + +if __name__ == '__main__': + main() diff --git a/utils/stack-usage-stats.sh b/utils/call_stacks_analysis/stack_usage_stats.sh similarity index 74% rename from utils/stack-usage-stats.sh rename to utils/call_stacks_analysis/stack_usage_stats.sh index 46e4090c40a..39f2013ee3b 100755 --- a/utils/stack-usage-stats.sh +++ b/utils/call_stacks_analysis/stack_usage_stats.sh @@ -3,8 +3,7 @@ # Copyright 2023, Intel Corporation # # -# utils/stack-usage-stat.sh -- combine stack usage into a file -# The script shall be run from the main PMDK folder. +# Combine stack usage into a file. The script shall be run from the main PMDK folder. # if [ ! -d "src/stats" ]; then