From 77b50b70b1f91ffc43162b6d0346b40753c2dd62 Mon Sep 17 00:00:00 2001 From: "Jan Winkelmann (keks)" Date: Tue, 27 May 2025 16:29:46 +0200 Subject: [PATCH] address feedback --- .github/workflows/bench-primitives.yml | 22 +++----- .github/workflows/bench-protocol.yml | 17 ++---- ciphers/benches/primitives.rs | 2 - flake.nix | 8 +++ readme.md | 73 ++++++++++++++------------ rosenpass/benches/trace_handshake.rs | 6 +-- 6 files changed, 62 insertions(+), 66 deletions(-) diff --git a/.github/workflows/bench-primitives.yml b/.github/workflows/bench-primitives.yml index 395bdea..d35855c 100644 --- a/.github/workflows/bench-primitives.yml +++ b/.github/workflows/bench-primitives.yml @@ -37,14 +37,6 @@ jobs: # Set up environment - - name: 🛠️ Config Linux x64 - run: echo "RUST_TARGET_FLAG=--target=x86_64-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.system == 'x86_64-linux' }} - - - name: 🛠️ Config Linux x86 - run: echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.system == 'i686-linux' }} - - name: 🛠️ Prepare Benchmark Path env: EVENT_NAME: ${{ github.event_name }} @@ -67,7 +59,7 @@ jobs: - name: 🏃🏻‍♀️ Benchmarks (using Nix as shell) working-directory: ciphers - run: nix develop ".#devShells.${{ matrix.system }}.default" --command cargo bench -F bench --bench primitives --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee ../bench-primitives.txt + run: nix develop ".#devShells.${{ matrix.system }}.benchmarks" --command cargo bench -F bench --bench primitives --verbose -- --output-format bencher | tee ../bench-primitives.txt - name: Extract benchmarks uses: cryspen/benchmark-data-extract-transform@v2 @@ -76,22 +68,24 @@ jobs: tool: "cargo" os: ${{ matrix.system }} output-file-path: bench-primitives.txt - data-out-path: bench-primitives.json + data-out-path: bench-primitives-os.json + + - name: Fix up 'os' label in benchmark data + run: jq 'map(with_entries(.key |= if . == "os" then "operating system" else . end))' bench-primitives.json - name: Upload benchmarks uses: cryspen/benchmark-upload-and-plot-action@v3 with: name: Crypto Primitives Benchmarks - group-by: "os,primitive,algorithm" - schema: "os,primitive,algorithm,implementation,operation,length" + group-by: "operating system,primitive,algorithm" + schema: "operating system,primitive,algorithm,implementation,operation,length" input-data-path: bench-primitives.json github-token: ${{ secrets.GITHUB_TOKEN }} # NOTE: pushes to current repository gh-repository: github.com/${{ github.repository }} - # use the default (gh-pages) for the demo - #gh-pages-branch: benchmarks auto-push: true fail-on-alert: true + base-path: benchmarks/ ciphers-primitives-bench-status: if: ${{ always() }} diff --git a/.github/workflows/bench-protocol.yml b/.github/workflows/bench-protocol.yml index 752ed90..f531651 100644 --- a/.github/workflows/bench-protocol.yml +++ b/.github/workflows/bench-protocol.yml @@ -37,14 +37,6 @@ jobs: # Set up environment - - name: 🛠️ Config Linux x64 - run: echo "RUST_TARGET_FLAG=--target=x86_64-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.system == 'x86_64-linux' }} - - - name: 🛠️ Config Linux x86 - run: echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.system == 'i686-linux' }} - - name: 🛠️ Prepare Benchmark Path env: EVENT_NAME: ${{ github.event_name }} @@ -66,22 +58,21 @@ jobs: # Benchmarks ... - name: 🏃🏻‍♀️ Benchmarks - run: nix develop ".#devShells.${{ matrix.system }}.default" --command cargo bench -p rosenpass --bench trace_handshake -F trace_bench --verbose $RUST_TARGET_FLAG >bench-protocol.json + run: nix develop ".#devShells.${{ matrix.system }}.benchmarks" --command cargo bench -p rosenpass --bench trace_handshake -F trace_bench --verbose >bench-protocol.json - name: Upload benchmarks uses: cryspen/benchmark-upload-and-plot-action@v3 with: name: Protocol Benchmarks - group-by: "os,arch,protocol version,run time" - schema: "os,arch,protocol version,run time,name" + group-by: "operating system,architecture,protocol version,run time" + schema: "operating system,architecture,protocol version,run time,name" input-data-path: bench-protocol.json github-token: ${{ secrets.GITHUB_TOKEN }} # NOTE: pushes to current repository gh-repository: github.com/${{ github.repository }} - # use the default (gh-pages) for the demo - #gh-pages-branch: benchmarks auto-push: true fail-on-alert: true + base-path: benchmarks/ ciphers-protocol-bench-status: if: ${{ always() }} diff --git a/ciphers/benches/primitives.rs b/ciphers/benches/primitives.rs index eb81c66..61a649b 100644 --- a/ciphers/benches/primitives.rs +++ b/ciphers/benches/primitives.rs @@ -342,5 +342,3 @@ mod keyed_hash { }); } } - -mod templates {} diff --git a/flake.nix b/flake.nix index 8ae3fad..9e2a935 100644 --- a/flake.nix +++ b/flake.nix @@ -173,6 +173,14 @@ inherit (pkgs.cargo-llvm-cov) LLVM_COV LLVM_PROFDATA; }; }; + devShells.benchmarks = pkgs.mkShell { + inputsFrom = [ pkgs.rosenpass ]; + nativeBuildInputs = with pkgs; [ + cargo-release + clippy + rustfmt + ]; + }; checks = { diff --git a/readme.md b/readme.md index c4ceb1a..c13782b 100644 --- a/readme.md +++ b/readme.md @@ -1,37 +1,3 @@ -# Changes on This Branch - -This branch adds facilities for benchmarking both the Rosenpass protocol -code and the implementations of the primitives behind it. The primitives -are benchmarked using criterion. For the protocol code, we use a custom -library for instrumenting the code such that events are written to a -trace, which is then inspected after a run. - -## Protocol Benchmark - -The trace that is being written to lives in a new module -`trace_bench` in the util crate. A basic benchmark that -performs some minor statistical analysis of the trace can be run using - -``` -cargo bench -p rosenpass --bench trace_handshake -F trace_bench -``` - -## Primitive Benchmark - -Benchmarks for the functions of the traits `Kem`, `Aead` and `KeyedHash` -have been added and are run for all implementations in the `primitives` -benchmark of `rosenpass-ciphers`. Run the benchmarks using - -``` -cargo bench -p rosenpass-ciphers --bench primitives -F bench -``` - -Note that the `bench` feature enables the inclusion of the libcrux-backed -trait implementations in the module tree, but does not enable them -as default. - ---- - # Rosenpass README ![Nix](https://github.com/rosenpass/rosenpass/actions/workflows/nix.yaml/badge.svg) @@ -117,6 +83,45 @@ Rosenpass is also available as prebuilt Docker images: For details on how to use these images, refer to the [Docker usage guide](docker/USAGE.md). +## Benchmarks + +This repository contains facilities for benchmarking both the Rosenpass +protocol code and the implementations of the cryptographic primitives used +by it. The primitives are benchmarked using criterion. For the protocol code +benchmarks we use a library for instrumenting the code such that events are +written to a trace, which is then inspected after a run. + +Benchmarks are automatically run on CI. The measurements are visualized in the +[Benchmark Dashboard]. + +[Benchmark Dashboard]: https://rosenpass.github.io/benchmarks + +### Primitive Benchmarks + +There are benchmarks for the functions of the traits `Kem`, `Aead` and +`KeyedHash`. They are run for all implementations in the `primitives` +benchmark of `rosenpass-ciphers`. Run the benchmarks using + +``` +cargo bench -p rosenpass-ciphers --bench primitives -F bench +``` + +Note that the `bench` feature enables the inclusion of the libcrux-backed +trait implementations in the module tree, but does not enable them +as default. + +### Protocol Benchmarks + +The trace that is being written to lives in a new module +`trace_bench` in the util crate. A basic benchmark that +performs some minor statistical analysis of the trace can be run using + +``` +cargo bench -p rosenpass --bench trace_handshake -F trace_bench +``` + +--- + # Mirrors Don't want to use GitHub or only have an IPv6 connection? Rosenpass has set up two mirrors for this: diff --git a/rosenpass/benches/trace_handshake.rs b/rosenpass/benches/trace_handshake.rs index 9adaafe..a8f6d16 100644 --- a/rosenpass/benches/trace_handshake.rs +++ b/rosenpass/benches/trace_handshake.rs @@ -183,8 +183,8 @@ impl std::fmt::Display for RunTimeGroup { let txt = match self { RunTimeGroup::Long => "long", RunTimeGroup::Medium => "medium", - RunTimeGroup::BelowMillisec => "below_ms", - RunTimeGroup::BelowMicrosec => "below_us", + RunTimeGroup::BelowMillisec => "below 1ms", + RunTimeGroup::BelowMicrosec => "below 1us", }; write!(f, "{txt}") } @@ -357,7 +357,7 @@ impl AggregateStat { // Format the JSON string using measured values and environment constants writeln!( w, - r#"{{"name":"{name}", "unit":"ns/iter", "value":"{value}", "range":"± {range}", "protocol version":"{protocol_version}", "sample size":"{sample_size}", "os":"{os}", "arch":"{arch}", "run time":"{run_time}"}}"#, + r#"{{"name":"{name}", "unit":"ns/iter", "value":"{value}", "range":"± {range}", "protocol version":"{protocol_version}", "sample size":"{sample_size}", "operating system":"{os}", "architecture":"{arch}", "run time":"{run_time}"}}"#, name = label, // Benchmark name value = self.mean_duration.as_nanos(), // Mean duration in nanoseconds range = self.sd_duration.as_nanos(), // Standard deviation in nanoseconds