address feedback

.github/workflows/bench-primitives.yml

@@ -37,14 +37,6 @@ jobs:
 
       # Set up environment
 
-      - name: 🛠️ Config Linux x64
-        run: echo "RUST_TARGET_FLAG=--target=x86_64-unknown-linux-gnu"  > $GITHUB_ENV
-        if: ${{ matrix.system == 'x86_64-linux' }}
-
-      - name: 🛠️ Config Linux x86
-        run: echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu"    > $GITHUB_ENV
-        if: ${{ matrix.system == 'i686-linux' }}
-
       - name: 🛠️ Prepare Benchmark Path
         env:
           EVENT_NAME:  ${{ github.event_name }}
@@ -67,7 +59,7 @@ jobs:
 
       - name: 🏃🏻‍♀️ Benchmarks (using Nix as shell)
         working-directory: ciphers
-        run: nix develop ".#devShells.${{ matrix.system }}.default" --command cargo bench -F bench --bench primitives --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee ../bench-primitives.txt
+        run: nix develop ".#devShells.${{ matrix.system }}.benchmarks" --command cargo bench -F bench --bench primitives --verbose -- --output-format bencher | tee ../bench-primitives.txt
 
       - name: Extract benchmarks
         uses: cryspen/benchmark-data-extract-transform@v2
@@ -76,22 +68,24 @@ jobs:
           tool: "cargo"
           os: ${{ matrix.system }}
           output-file-path: bench-primitives.txt
-          data-out-path: bench-primitives.json
+          data-out-path: bench-primitives-os.json
+
+      - name: Fix up 'os' label in benchmark data
+        run: jq 'map(with_entries(.key |= if . == "os" then "operating system" else . end))' <bench-primitives-os.json >bench-primitives.json
 
       - name: Upload benchmarks
         uses: cryspen/benchmark-upload-and-plot-action@v3
         with:
           name: Crypto Primitives Benchmarks
-          group-by: "os,primitive,algorithm"
+          group-by: "operating system,primitive,algorithm"
-          schema: "os,primitive,algorithm,implementation,operation,length"
+          schema: "operating system,primitive,algorithm,implementation,operation,length"
           input-data-path: bench-primitives.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
           # NOTE: pushes to current repository
           gh-repository: github.com/${{ github.repository }}
-          # use the default (gh-pages) for the demo
-          #gh-pages-branch: benchmarks
           auto-push: true
           fail-on-alert: true
+          base-path: benchmarks/
 
   ciphers-primitives-bench-status:
     if: ${{ always() }}

.github/workflows/bench-protocol.yml

@@ -37,14 +37,6 @@ jobs:
 
       # Set up environment
 
-      - name: 🛠️ Config Linux x64
-        run: echo "RUST_TARGET_FLAG=--target=x86_64-unknown-linux-gnu"  > $GITHUB_ENV
-        if: ${{ matrix.system == 'x86_64-linux' }}
-
-      - name: 🛠️ Config Linux x86
-        run: echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu"    > $GITHUB_ENV
-        if: ${{ matrix.system == 'i686-linux' }}
-
       - name: 🛠️ Prepare Benchmark Path
         env:
           EVENT_NAME:  ${{ github.event_name }}
@@ -66,22 +58,21 @@ jobs:
       # Benchmarks ...
 
       - name: 🏃🏻‍♀️ Benchmarks
-        run: nix develop ".#devShells.${{ matrix.system }}.default" --command cargo bench -p rosenpass --bench trace_handshake -F trace_bench --verbose $RUST_TARGET_FLAG >bench-protocol.json
+        run: nix develop ".#devShells.${{ matrix.system }}.benchmarks" --command cargo bench -p rosenpass --bench trace_handshake -F trace_bench --verbose >bench-protocol.json
 
       - name: Upload benchmarks
         uses: cryspen/benchmark-upload-and-plot-action@v3
         with:
           name: Protocol Benchmarks
-          group-by: "os,arch,protocol version,run time"
+          group-by: "operating system,architecture,protocol version,run time"
-          schema: "os,arch,protocol version,run time,name"
+          schema: "operating system,architecture,protocol version,run time,name"
           input-data-path: bench-protocol.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
           # NOTE: pushes to current repository
           gh-repository: github.com/${{ github.repository }}
-          # use the default (gh-pages) for the demo
-          #gh-pages-branch: benchmarks
           auto-push: true
           fail-on-alert: true
+          base-path: benchmarks/
 
   ciphers-protocol-bench-status:
     if: ${{ always() }}

ciphers/benches/primitives.rs

@@ -342,5 +342,3 @@ mod keyed_hash {
         });
     }
 }
-
-mod templates {}

flake.nix

@@ -173,6 +173,14 @@
                 inherit (pkgs.cargo-llvm-cov) LLVM_COV LLVM_PROFDATA;
               };
             };
+            devShells.benchmarks = pkgs.mkShell {
+              inputsFrom = [ pkgs.rosenpass ];
+              nativeBuildInputs = with pkgs; [
+                cargo-release
+                clippy
+                rustfmt
+              ];
+            };
 
             checks =
               {

readme.md

@@ -1,37 +1,3 @@
-# Changes on This Branch
-
-This branch adds facilities for benchmarking both the Rosenpass protocol
-code and the implementations of the primitives behind it. The primitives
-are benchmarked using criterion. For the protocol code, we use a custom
-library for instrumenting the code such that events are written to a
-trace, which is then inspected after a run.
-
-## Protocol Benchmark
-
-The trace that is being written to lives in a new module
-`trace_bench` in the util crate. A basic benchmark that
-performs some minor statistical analysis of the trace can be run using
-
-```
-cargo bench -p rosenpass --bench trace_handshake -F trace_bench
-```
-
-## Primitive Benchmark
-
-Benchmarks for the functions of the traits `Kem`, `Aead` and `KeyedHash`
-have been added and are run for all implementations in the `primitives`
-benchmark of `rosenpass-ciphers`. Run the benchmarks using
-
-```
-cargo bench -p rosenpass-ciphers --bench primitives -F bench
-```
-
-Note that the `bench` feature enables the inclusion of the libcrux-backed
-trait implementations in the module tree, but does not enable them
-as default.
-
----
-
 # Rosenpass README
 
 ![Nix](https://github.com/rosenpass/rosenpass/actions/workflows/nix.yaml/badge.svg)
@@ -117,6 +83,45 @@ Rosenpass is also available as prebuilt Docker images:
 
 For details on how to use these images, refer to the [Docker usage guide](docker/USAGE.md).
 
+## Benchmarks
+
+This repository contains facilities for benchmarking both the Rosenpass
+protocol code and the implementations of the cryptographic primitives used
+by it. The primitives are benchmarked using criterion. For the protocol code
+benchmarks we use a library for instrumenting the code such that events are
+written to a trace, which is then inspected after a run.
+
+Benchmarks are automatically run on CI. The measurements are visualized in the
+[Benchmark Dashboard].
+
+[Benchmark Dashboard]: https://rosenpass.github.io/benchmarks
+ 
+### Primitive Benchmarks
+
+There are benchmarks for the functions of the traits `Kem`, `Aead` and
+`KeyedHash`. They are run for all implementations in the `primitives`
+benchmark of `rosenpass-ciphers`. Run the benchmarks using
+
+```
+cargo bench -p rosenpass-ciphers --bench primitives -F bench
+```
+
+Note that the `bench` feature enables the inclusion of the libcrux-backed
+trait implementations in the module tree, but does not enable them
+as default.
+
+### Protocol Benchmarks
+
+The trace that is being written to lives in a new module
+`trace_bench` in the util crate. A basic benchmark that
+performs some minor statistical analysis of the trace can be run using
+
+```
+cargo bench -p rosenpass --bench trace_handshake -F trace_bench
+```
+
+---
+
 # Mirrors
 
 Don't want to use GitHub or only have an IPv6 connection? Rosenpass has set up two mirrors for this:

rosenpass/benches/trace_handshake.rs

@@ -183,8 +183,8 @@ impl std::fmt::Display for RunTimeGroup {
         let txt = match self {
             RunTimeGroup::Long => "long",
             RunTimeGroup::Medium => "medium",
-            RunTimeGroup::BelowMillisec => "below_ms",
+            RunTimeGroup::BelowMillisec => "below 1ms",
-            RunTimeGroup::BelowMicrosec => "below_us",
+            RunTimeGroup::BelowMicrosec => "below 1us",
         };
         write!(f, "{txt}")
     }
@@ -357,7 +357,7 @@ impl AggregateStat<Duration> {
         // Format the JSON string using measured values and environment constants
         writeln!(
             w,
-            r#"{{"name":"{name}", "unit":"ns/iter", "value":"{value}", "range":"± {range}", "protocol version":"{protocol_version}", "sample size":"{sample_size}", "os":"{os}", "arch":"{arch}", "run time":"{run_time}"}}"#,
+            r#"{{"name":"{name}", "unit":"ns/iter", "value":"{value}", "range":"± {range}", "protocol version":"{protocol_version}", "sample size":"{sample_size}", "operating system":"{os}", "architecture":"{arch}", "run time":"{run_time}"}}"#,
             name = label,                          // Benchmark name
             value = self.mean_duration.as_nanos(), // Mean duration in nanoseconds
             range = self.sd_duration.as_nanos(),   // Standard deviation in nanoseconds