diff --git a/pyproject.toml b/pyproject.toml index 2041577..696d974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.218" +version = "0.1.219" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index c35c6e5..3d24498 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -96,6 +96,9 @@ def create_compare_arguments(parser): parser.add_argument( "--baseline-target-version", type=str, default=None, required=False ) + parser.add_argument( + "--baseline-target-branch", type=str, default=None, required=False + ) parser.add_argument("--comparison-branch", type=str, default=None, required=False) parser.add_argument( "--baseline-github-repo", type=str, default="redis", required=False @@ -108,6 +111,9 @@ def create_compare_arguments(parser): parser.add_argument( "--comparison-target-version", type=str, default=None, required=False ) + parser.add_argument( + "--comparison-target-branch", type=str, default=None, required=False + ) parser.add_argument("--print-regressions-only", type=bool, default=False) parser.add_argument("--print-improvements-only", type=bool, default=False) parser.add_argument("--skip-unstable", type=bool, default=False) diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index af72d69..13b2ff0 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -203,6 +203,8 @@ def compare_command_logic(args, project_name, project_version): ) ) baseline_branch = default_baseline_branch + if baseline_branch == "": + baseline_branch = None comparison_branch = args.comparison_branch simplify_table = args.simple_table print_regressions_only = args.print_regressions_only @@ -250,6 +252,8 @@ def compare_command_logic(args, project_name, project_version): running_platform = args.running_platform baseline_target_version = args.baseline_target_version comparison_target_version = args.comparison_target_version + baseline_target_branch = args.baseline_target_branch + comparison_target_branch = args.comparison_target_branch baseline_github_repo = args.baseline_github_repo comparison_github_repo = args.comparison_github_repo baseline_hash = args.baseline_hash @@ -318,6 +322,8 @@ def compare_command_logic(args, project_name, project_version): comparison_hash, baseline_github_repo, comparison_github_repo, + baseline_target_branch, + comparison_target_branch, ) prepare_regression_comment( auto_approve, @@ -547,6 +553,8 @@ def compute_regression_table( baseline_hash=None, baseline_github_repo="redis", comparison_github_repo="redis", + baseline_target_branch=None, + comparison_target_branch=None, ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -574,6 +582,8 @@ def compute_regression_table( comparison_target_version, comparison_hash, baseline_hash, + baseline_target_branch, + comparison_target_branch, ) logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}") logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}") @@ -739,6 +749,11 @@ def compute_regression_table( ) +def get_by_error(name, by_str_arr): + by_string = ",".join(by_str_arr) + return f"--{name}-branch, --{name}-tag, --{name}-target-branch, --{name}-hash, and --{name}-target-version are mutually exclusive. You selected a total of {len(by_str_arr)}: {by_string}. Pick one..." + + def get_by_strings( baseline_branch, comparison_branch, @@ -748,6 +763,8 @@ def get_by_strings( comparison_target_version=None, baseline_hash=None, comparison_hash=None, + baseline_target_branch=None, + comparison_target_branch=None, ): baseline_covered = False comparison_covered = False @@ -755,50 +772,64 @@ def get_by_strings( by_str_comparison = "" baseline_str = "" comparison_str = "" + baseline_by_arr = [] + comparison_by_arr = [] + + ################# BASELINE BY .... + if baseline_branch is not None: - baseline_covered = True by_str_baseline = "branch" + baseline_covered = True baseline_str = baseline_branch - if comparison_branch is not None: - comparison_covered = True - by_str_comparison = "branch" - comparison_str = comparison_branch + baseline_by_arr.append(by_str_baseline) if baseline_tag is not None: - if comparison_covered: - logging.error( - "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..." - ) + by_str_baseline = "version" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "version" baseline_str = baseline_tag if baseline_target_version is not None: - if comparison_covered: - logging.error( - "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..." - ) + by_str_baseline = "target+version" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "target+version" baseline_str = baseline_target_version if baseline_hash is not None: - if comparison_covered: - logging.error( - "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..." - ) + by_str_baseline = "hash" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "hash" baseline_str = baseline_hash + if baseline_target_branch is not None: + by_str_baseline = "target+branch" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) + exit(1) + baseline_covered = True + baseline_str = baseline_target_branch + + ################# COMPARISON BY .... + + if comparison_branch is not None: + by_str_comparison = "branch" + comparison_covered = True + comparison_str = comparison_branch if comparison_tag is not None: # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch and --comparison-tag are mutually exclusive. Pick one..." + "--comparison-branch and --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True @@ -808,18 +839,29 @@ def get_by_strings( # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..." + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True by_str_comparison = "target+version" comparison_str = comparison_target_version + if comparison_target_branch is not None: + # check if we had already covered comparison + if comparison_covered: + logging.error( + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." + ) + exit(1) + comparison_covered = True + by_str_comparison = "target+branch" + comparison_str = comparison_target_branch + if comparison_hash is not None: # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..." + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True @@ -829,13 +871,13 @@ def get_by_strings( if baseline_covered is False: logging.error( "You need to provider either " - + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) " + + "( --baseline-branch, --baseline-tag, --baseline-hash, --baseline-target-branch or --baseline-target-version ) " ) exit(1) if comparison_covered is False: logging.error( "You need to provider either " - + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) " + + "( --comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch or --comparison-target-version ) " ) exit(1) return baseline_str, by_str_baseline, comparison_str, by_str_comparison @@ -1252,9 +1294,9 @@ def get_v_pct_change_and_largest_var( if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n): comparison_values.append(tuple[1]) comparison_df = pd.DataFrame(comparison_values) - comparison_median = float(comparison_df.median()) + comparison_median = float(comparison_df.median().iloc[0]) comparison_v = comparison_median - comparison_std = float(comparison_df.std()) + comparison_std = float(comparison_df.std().iloc[0]) if verbose: logging.info( "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format( diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml new file mode 100644 index 0000000..c1785e2 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-geo-2-elements-geopos +description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geopos, and we query it using GEOPOS command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"' +tested-groups: +- geo +tested-commands: +- geopos +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="GEOPOS Sicily Palermo Catania" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 138 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml new file mode 100644 index 0000000..04d37ed --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord +description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geosearch, and we query it using GEOSEARCH command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"' + - '"GEOADD" "Sicily" "12.758489" "38.788135" "edge1" "17.241510" "38.788135" "edge2"' +tested-groups: +- geo +tested-commands: +- geosearch +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="GEOSEARCH Sicily FROMLONLAT 15 37 BYBOX 400 400 km ASC WITHCOORD WITHDIST" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 138 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml new file mode 100644 index 0000000..6f2eba0 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-set-100-elements-sismember-is-a-member +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is a member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- set +tested-commands: +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:100 lysbgqqfqw" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml new file mode 100644 index 0000000..ae1e511 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-set-100-elements-sismember-not-a-member +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is not a member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- set +tested-commands: +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:100 not-a-member" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml new file mode 100644 index 0000000..76ec81e --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 10M elements. We query it using SISMEMBER in which the value has 50% change of being member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SADD set:10M:elements __key__" --command-key-pattern=P --key-maximum 10000000 --key-prefix "" -n 10000000 --hide-histogram -t 1 -c 1 +tested-groups: +- set +tested-commands: +- sadd +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:10M:elements __key__" --key-maximum 20000000 --key-prefix "" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml new file mode 100644 index 0000000..03cb27d --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 1M elements. We query it using SISMEMBER in which the value has 50% change of being member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SADD set:1M:elements __key__" --command-key-pattern=P --key-maximum 1000000 --key-prefix "" -n 1000000 --hide-histogram -t 1 -c 1 +tested-groups: +- set +tested-commands: +- sadd +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:1M:elements __key__" --key-maximum 2000000 --key-prefix "" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml new file mode 100644 index 0000000..9db858f --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml @@ -0,0 +1,22 @@ +version: 0.4 +name: memtier_benchmark-connection-hello +description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.' +tested-groups: +- connection +tested-commands: +- hello +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="HELLO 2 SETNAME __key__" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1