diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e95b8ec1d..5e16e5db9 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -731,6 +731,33 @@ jobs: system_name: tioga system_spec: llnl-elcapitan rocm=5.5.1 compiler=cce +gtl + - name: hpcg/openmp caliper=mpi,time tioga llnl-elcapitan compiler=cce + uses: ./.github/actions/dynamic-dry-run + with: + benchmark_name: hpcg + benchmark_mode: openmp + benchmark_spec: hpcg+openmp caliper=mpi,time + system_name: tioga + system_spec: llnl-elcapitan compiler=cce + + - name: hpcg/openmp caliper=time lassen llnl-sierra compiler=clang-ibm + uses: ./.github/actions/dynamic-dry-run + with: + benchmark_name: hpcg + benchmark_mode: openmp + benchmark_spec: hpcg+openmp caliper=mpi,time + system_name: lassen + system_spec: llnl-sierra compiler=clang-ibm + + - name: hpcg/openmp caliper=cuda,time ruby llnl-cluster cluster=ruby compiler=gcc + uses: ./.github/actions/dynamic-dry-run + with: + benchmark_name: hpcg + benchmark_mode: openmp + benchmark_spec: hpcg+openmp caliper=mpi,time + system_name: ruby + system_spec: llnl-cluster cluster=ruby compiler=gcc + - name: hpl/mpi caliper=mpi,time tioga llnl-elcapitan rocm=5.5.1 compiler=cce +gtl blas=intel-oneapi-mkl uses: ./.github/actions/dynamic-dry-run with: diff --git a/experiments/hpcg/experiment.py b/experiments/hpcg/experiment.py new file mode 100644 index 000000000..8056362f5 --- /dev/null +++ b/experiments/hpcg/experiment.py @@ -0,0 +1,99 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from benchpark.directives import variant +from benchpark.experiment import Experiment +from benchpark.scaling import StrongScaling +from benchpark.scaling import WeakScaling +from benchpark.openmp import OpenMPExperiment +from benchpark.caliper import Caliper + + +class Hpcg( + Experiment, + StrongScaling, + WeakScaling, + OpenMPExperiment, + Caliper, +): + + variant( + "workload", + default="standard", + description="workload to run", + ) + variant( + "version", + default="caliper", + values=("3.1", "develop", "caliper"), + description="app version", + ) + + def compute_applications_section(self): + + problem_sizes = {"mx": 104, "my": 104, "mz": 104} + num_procs = {"x": 1, "y": 1, "z": 1} + n_resources = 1 + + if self.spec.satisfies("+single_node"): + for k, v in problem_sizes.items(): + self.add_experiment_variable(k, v, True) + + elif self.spec.satisfies("+strong"): + scaled_variables = self.generate_strong_scaling_params( + {tuple(num_procs.keys()): list(num_procs.values())}, + int(self.spec.variants["scaling-factor"][0]), + int(self.spec.variants["scaling-iterations"][0]), + ) + n_resources = [ + x * y * z + for x, y, z in zip( + *(scaled_variables[p] for p in num_procs if p in scaled_variables) + ) + ] + for k, v in problem_sizes.items(): + self.add_experiment_variable(k, v, True) + + elif self.spec.satisfies("+weak"): + scaled_variables = self.generate_weak_scaling_params( + {tuple(num_procs.keys()): list(num_procs.values())}, + {tuple(problem_sizes.keys()): list(problem_sizes.values())}, + int(self.spec.variants["scaling-factor"][0]), + int(self.spec.variants["scaling-iterations"][0]), + ) + n_resources = [ + x * y * z + for x, y, z in zip( + *(scaled_variables[p] for p in num_procs if p in scaled_variables) + ) + ] + for k, v in scaled_variables.items(): + if k in problem_sizes: + self.add_experiment_variable(k, v, True) + + self.add_experiment_variable("n_ranks", n_resources, True) + self.add_experiment_variable("n_threads_per_proc", 1, True) + self.add_experiment_variable("matrix_size", "{mx} {my} {mz}", False) + + self.add_experiment_variable("iterations", "60", False) + + def compute_spack_section(self): + # get package version + app_version = self.spec.variants["version"][0] + + # get system config options + # TODO: Get compiler/mpi/package handles directly from system.py + system_specs = {} + system_specs["compiler"] = "default-compiler" + system_specs["mpi"] = "default-mpi" + + # set package spack specs + # empty package_specs value implies external package + self.add_spack_spec(system_specs["mpi"]) + # self.add_spack_spec(system_specs["blas"]) + + self.add_spack_spec( + self.name, [f"hpcg@{app_version}", system_specs["compiler"]] + ) diff --git a/repo/hpcg/application.py b/repo/hpcg/application.py index aa88eb534..2fcf1e720 100644 --- a/repo/hpcg/application.py +++ b/repo/hpcg/application.py @@ -3,16 +3,128 @@ # # SPDX-License-Identifier: Apache-2.0 -import sys - +import os from ramble.appkit import * -from ramble.app.builtin.hpcg import Hpcg as HpcgBase +from ramble.expander import Expander + + +class Hpcg(ExecutableApplication): + """Define HPCG application""" + name = "hpcg" -class Hpcg(HpcgBase): + maintainers("douglasjacobsen") tags = ['synthetic', 'conjugate-gradient','solver','sparse-linear-algebra', 'large-scale', 'mpi','network-point-to-point', 'c++','openmp'] + + executable("execute", "xhpcg", use_mpi=True) + + executable("move-log", "mv {experiment_run_dir}/HPCG-Benchmark*.txt {out_file}", use_mpi=False) + + workload("standard", executables=["execute", "move-log"]) + + workload_variable( + "matrix_size", + default="104 104 104", + description="Dimensions of the matrix to use", + workloads=["standard"], + ) + + workload_variable( + "iterations", + default="60", + description="Number of iterations to perform", + workloads=["standard"], + ) + + workload_variable( + "out_file", + default="{experiment_run_dir}/{experiment_name}.out", + description="Output file for results", + workloads=["standard"], + ) + + log_str = Expander.expansion_str("out_file") + + figure_of_merit( + "Status", + log_file=log_str, + fom_regex=r"Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)", + group_name="status", + units="", + ) + + figure_of_merit( + "Gflops", + log_file=log_str, + fom_regex=r"Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)", + group_name="gflops", + units="GFLOP/s", + ) + + figure_of_merit( + "Time", + log_file=log_str, + fom_regex=r"Final Summary::Results are.* execution time.*is=(?P[0-9]+\.[0-9]*)", + group_name="exec_time", + units="s", + ) + + figure_of_merit( + "ComputeDotProductMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeDotProduct used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeSPMVMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeSPMV used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeMGMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeMG used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeWAXPBYMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeWAXPBY used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "HPCG 2.4 Rating", + log_file=log_str, + fom_regex=r"Final Summary::HPCG 2\.4 rating.*=(?P[0-9]+\.*[0-9]*)", + group_name="rating", + units="", + ) + + def _make_experiments(self, workspace, app_inst=None): + super()._make_experiments(workspace) + + input_path = os.path.join( + self.expander.expand_var_name("experiment_run_dir"), "hpcg.dat" + ) + + with open(input_path, "w+") as f: + f.write("HPCG benchmark input file\n") + f.write( + "Sandia National Laboratories; University of Tennessee, Knoxville\n" + ) + f.write(self.expander.expand_var_name("matrix_size") + "\n") + f.write(self.expander.expand_var_name("iterations") + "\n") diff --git a/repo/hpcg/package.py b/repo/hpcg/package.py new file mode 100644 index 000000000..34e62c698 --- /dev/null +++ b/repo/hpcg/package.py @@ -0,0 +1,42 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import platform + +from spack.package import * + + +class Hpcg(CMakePackage): + """HPCG is a software package that performs a fixed number of multigrid + preconditioned (using a symmetric Gauss-Seidel smoother) conjugate gradient + (PCG) iterations using double precision (64 bit) floating point values.""" + + #homepage = "https://www.hpcg-benchmark.org" + #url = "https://www.hpcg-benchmark.org/downloads/hpcg-3.1.tar.gz" + git = "https://github.com/daboehme/hpcg.git" + + version("develop", branch="master") + #version("3.1", sha256="33a434e716b79e59e745f77ff72639c32623e7f928eeb7977655ffcaade0f4a4") + version("caliper", branch="caliper-support") + + variant("openmp", default=True, description="Enable OpenMP support") + variant("caliper", default=False, description="Enable Caliper support") + + depends_on("mpi@1.1:") + depends_on("caliper", when="+caliper") + depends_on("adiak", when="+caliper") + + + def cmake_args(self): + build_targets = ["all", "docs"] + install_targets = ["install", "docs"] + args = [ + "-DHPCG_ENABLE_MPI=TRUE", + self.define_from_variant("HPCG_ENABLE_CALIPER", "caliper"), + self.define_from_variant("HPCG_ENABLE_OPENMP", "openmp"), + ] + + return args