add BenchmarkCI (#269)

* add `BenchmarkCI`

* fix branch error

* Format .jl files (#270)

Co-authored-by: prbzrg <[email protected]>

* rename

* fix benchmark

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: prbzrg <[email protected]>

.github/workflows/Benchmark.yml

@@ -0,0 +1,25 @@
+name: Benchmarks
+on: pull_request
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
+          arch: x64
+          show-versioninfo: true
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@v1
+      - name: Install dependencies
+        run: julia --color=yes -e 'using Pkg; Pkg.add(["PkgBenchmark", "BenchmarkCI"])'
+      - name: Run benchmarks
+        run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.judge(; baseline="origin/main")'
+      - name: Print judgement
+        run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()'
+      - name: Post results
+        run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.postjudge()'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CI.yml

@@ -16,7 +16,6 @@ jobs:
           - RNODE
           - FFJORD
           - Planar
-          - Benchmark
           - Quality
           - Instability
         version:

.gitignore

@@ -25,3 +25,6 @@ Manifest.toml
 LocalPreferences.toml
 
 .vscode
+
+.benchmarkci
+benchmark/*.json

benchmark/Project.toml

@@ -0,0 +1,7 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

benchmark/benchmarks.jl

@@ -0,0 +1,37 @@
+using ContinuousNormalizingFlows, BenchmarkTools, Flux, Lux, PkgBenchmark, Random, Zygote
+
+SUITE = BenchmarkGroup()
+
+SUITE["main"] = BenchmarkGroup(["package", "simple"])
+SUITE["main"]["0-order"] = BenchmarkGroup(["direct"])
+SUITE["main"]["1-order"] = BenchmarkGroup(["gradient"])
+
+BenchmarkTools.DEFAULT_PARAMETERS.samples = 2^13
+BenchmarkTools.DEFAULT_PARAMETERS.seconds = convert(Float64, 2 * 60)
+BenchmarkTools.DEFAULT_PARAMETERS.evals = 1
+BenchmarkTools.DEFAULT_PARAMETERS.overhead = 0.0
+BenchmarkTools.DEFAULT_PARAMETERS.gctrial = true
+BenchmarkTools.DEFAULT_PARAMETERS.gcsample = false
+BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = convert(Float64, eps(Float16))
+BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = convert(Float64, eps(Float16))
+
+nvars = 8
+n = 128
+rng = Random.default_rng()
+r = rand(Float32, nvars, n)
+
+nn = FluxCompatLayer(Flux.f32(Flux.Dense(nvars => nvars, tanh)))
+icnf = construct(RNODE, nn, nvars; compute_mode = ZygoteMatrixMode)
+
+ps, st = Lux.setup(rng, icnf)
+diff_loss_train(x) = loss(icnf, TrainMode(), r, x, st)
+diff_loss_test(x) = loss(icnf, TestMode(), r, x, st)
+grad_diff_loss_train() = Zygote.gradient(diff_loss_train, ps)
+grad_diff_loss_test() = Zygote.gradient(diff_loss_test, ps)
+t_loss_train() = loss(icnf, TrainMode(), r, ps, st)
+t_loss_test() = loss(icnf, TestMode(), r, ps, st)
+
+SUITE["main"]["0-order"]["train"] = @benchmarkable $t_loss_train()
+SUITE["main"]["0-order"]["test"] = @benchmarkable $t_loss_test()
+SUITE["main"]["1-order"]["train"] = @benchmarkable $grad_diff_loss_train()
+SUITE["main"]["1-order"]["test"] = @benchmarkable $grad_diff_loss_test()

test/Project.toml

@@ -2,7 +2,6 @@
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"

test/runtests.jl

@@ -3,7 +3,6 @@ using Test
 using AbstractDifferentiation: AbstractDifferentiation
 using ADTypes: ADTypes
 using Aqua: Aqua
-using BenchmarkTools: BenchmarkTools
 using ComponentArrays: ComponentArrays
 using ComputationalResources: ComputationalResources
 using CUDA: CUDA
@@ -42,10 +41,6 @@ USE_GPU = get(ENV, "USE_GPU", "Yes") == "Yes"
         end
     end
 
-    if GROUP == "All" || GROUP == "Benchmark"
-        include("benchmark_tests.jl")
-    end
-
     if GROUP == "All" || GROUP == "Quality"
         include("quality_tests.jl")
     end