refactor(flake.nix): externalize pkgs, add overlay

This splits the complexity of the `flake.nix` into multiple files. At
cross-compiled and static builds at the benefit of simpler nix
expressions and generally better cross compilation compatibility.
the same time, naersk is removed; causing much slower builds for cross-
compiled packages.

This partially addresses the points mentioned in #412.

.ci/run-regression.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+iterations="$1"
+sleep_time="$2"
+
+PWD="$(pwd)"
+EXEC="$PWD/target/release/rosenpass"
+LOGS="$PWD/output/logs"
+
+mkdir -p "$LOGS"
+
+run_command() {
+  local file=$1
+  local log_file="$2"
+  ("$EXEC" exchange-config "$file" 2>&1 | tee -a "$log_file") &
+  echo $!
+}
+
+pids=()
+
+(cd output/dut && run_command "configs/dut-$iterations.toml" "$LOGS/dut.log")
+for (( x=0; x<iterations; x++ )); do
+  (cd output/ate && run_command "configs/ate-$x.toml" "$LOGS/ate-$x.log") & pids+=($!)
+done
+
+sleep "$sleep_time"
+
+lsof -i :9999 | awk 'NR!=1 {print $2}' | xargs kill
+
+for (( x=0; x<iterations; x++ )); do
+    port=$((x + 50000))
+    lsof -i :$port | awk 'NR!=1 {print $2}' | xargs kill
+done

.github/workflows/nix.yaml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   i686-linux---default:
     name: Build i686-linux.default
@@ -241,30 +246,30 @@ jobs:
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
       - name: Build
         run: nix build .#packages.x86_64-linux.release-package --print-build-logs
-  aarch64-linux---release-package:
-    name: Build aarch64-linux.release-package
-    runs-on:
-      - ubuntu-latest
-    needs:
-      - aarch64-linux---rosenpass-oci-image
-      - aarch64-linux---rosenpass
-      - aarch64-linux---rp
-    steps:
-      - run: |
-          DEBIAN_FRONTEND=noninteractive
-          sudo apt-get update -q -y && sudo apt-get install -q -y qemu-system-aarch64 qemu-efi binfmt-support qemu-user-static
-      - uses: actions/checkout@v3
-      - uses: cachix/install-nix-action@v22
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
-          extra_nix_config: |
-            system = aarch64-linux
-      - uses: cachix/cachix-action@v12
-        with:
-          name: rosenpass
-          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
-      - name: Build
-        run: nix build .#packages.aarch64-linux.release-package --print-build-logs
+  # aarch64-linux---release-package:
+  #   name: Build aarch64-linux.release-package
+  #   runs-on:
+  #     - ubuntu-latest
+  #   needs:
+  #     - aarch64-linux---rosenpass-oci-image
+  #     - aarch64-linux---rosenpass
+  #     - aarch64-linux---rp
+  #   steps:
+  #     - run: |
+  #         DEBIAN_FRONTEND=noninteractive
+  #         sudo apt-get update -q -y && sudo apt-get install -q -y qemu-system-aarch64 qemu-efi binfmt-support qemu-user-static
+  #     - uses: actions/checkout@v3
+  #     - uses: cachix/install-nix-action@v22
+  #       with:
+  #         nix_path: nixpkgs=channel:nixos-unstable
+  #         extra_nix_config: |
+  #           system = aarch64-linux
+  #     - uses: cachix/cachix-action@v12
+  #       with:
+  #         name: rosenpass
+  #         authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+  #     - name: Build
+  #       run: nix build .#packages.aarch64-linux.release-package --print-build-logs
   x86_64-linux---rosenpass:
     name: Build x86_64-linux.rosenpass
     runs-on:

.github/workflows/qc.yaml

@@ -4,6 +4,10 @@ on:
   push:
     branches: [main]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   checks: write
   contents: read
@@ -110,7 +114,12 @@ jobs:
       - run: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items
 
   cargo-test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-13]
+        # - ubuntu is x86-64
+        # - macos-13 is also x86-64 architecture
     steps:
       - uses: actions/checkout@v3
       - uses: actions/cache@v3
@@ -190,13 +199,16 @@ jobs:
       - run: rustup component add llvm-tools-preview
       - run: |
           cargo install cargo-llvm-cov || true
-          cargo llvm-cov --lcov --output-path coverage.lcov
+          cargo llvm-cov \
+            --workspace\
+            --all-features \
+            --lcov \
+            --output-path coverage.lcov
       # If using tarapulin
       #- run: cargo install cargo-tarpaulin
       #- run: cargo tarpaulin --out Xml
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.lcov
           verbose: true

.github/workflows/regressions.yml

@@ -0,0 +1,25 @@
+name: Regressions
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  checks: write
+  contents: read
+
+jobs:
+  multi-peer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: cargo build --bin rosenpass --release
+      - run: python misc/generate_configs.py
+      - run: chmod +x .ci/run-regression.sh
+      - run: .ci/run-regression.sh 100 20
+      - run: |
+          [ $(ls -1 output/ate/out | wc -l) -eq 100 ]

.gitignore

@@ -20,3 +20,8 @@ _markdown_*
 **/result
 **/result-*
 .direnv
+
+# Visual studio code
+.vscode
+
+/output

.mailmap

@@ -0,0 +1 @@
+Clara Engler <cve@cve.cx> <me@emilengler.com>

.prettierignore

@@ -1,4 +1,5 @@
 .direnv/
+flake.lock
 papers/whitepaper.md
-target/
 src/usage.md
+target/

CONTRIBUTING.md

@@ -0,0 +1,38 @@
+**Making a new Release of Rosenpass — Cooking Recipe**
+
+If you have to change a file, do what it takes to get the change as commit on the main branch, then **start from step 0**.
+If any other issue occurs
+
+0. Make sure you are in the root directory of the project
+   - `cd "$(git rev-parse --show-toplevel)"`
+1. Make sure you locally checked out the head of the main branch
+   - `git stash --include-untracked && git checkout main && git pull`
+2. Make sure all tests pass
+   - `cargo test`
+3. Make sure the current version in `rosenpass/Cargo.toml` matches that in the [last release on GitHub](https://github.com/rosenpass/rosenpass/releases)
+   - Only normal releases count, release candidates and draft releases can be ignored
+4. Pick the kind of release that you want to make (`major`, `minor`, `patch`, `rc`, ...)
+   - See `cargo release --help` for more information on the available release types
+   - Pick `rc` if in doubt
+5. Try to release a new version
+   - `cargo release rc --package rosenpass`
+   - An issue was reported? Go fix it, start again with step 0!
+6. Actually make the release
+   - `cargo release rc --package rosenpass --execute`
+   - Tentatively wait for any interactions, such as entering ssh keys etc.
+   - You may be asked for your ssh key multiple times!
+
+**Frequently Asked Questions (FAQ)**
+
+- You have untracked files, which `cargo release` complains about?
+  - `git stash --include-untracked`
+- You cannot push to crates.io because you are not logged in?
+  - Follow the steps displayed in [`cargo login`](https://doc.rust-lang.org/cargo/commands/cargo-login.html)
+- How is the release page added to [GitHub Releases](https://github.com/rosenpass/rosenpass/releases) itself?
+  - Our CI Pipeline will create the release, once `cargo release` pushed the new version tag to the repo. The new release should pop up almost immediately in [GitHub Releases](https://github.com/rosenpass/rosenpass/releases) after the [Actions/Release](https://github.com/rosenpass/rosenpass/actions/workflows/release.yaml) pipeline started.
+- No new release pops up in the `Release` sidebar element on the [main page](https://github.com/rosenpass/rosenpass)
+  - Did you push a `rc` release? This view only shows non-draft release, but `rc` releases are considered as draft. See [Releases](https://github.com/rosenpass/rosenpass/releases) page to see all (including draft!) releases.
+- The release page was created on GitHub, but there are no assets/artifacts other than the source code tar ball/zip?
+  - The artifacts are generated and pushed automatically to the release, but this takes some time (a couple of minutes). You can check the respective CI pipeline: [Actions/Release](https://github.com/rosenpass/rosenpass/actions/workflows/release.yaml), which should start immediately after `cargo release` pushed the new release tag to the repo. The release artifacts only are added later to the release, once all jobs in bespoke pipeline finished.
+- How are the release artifacts generated, and what are they?
+  - The release artifacts are built using one Nix derivation per platform, `nix build .#release-package`. It contains both statically linked versions of `rosenpass` itself and OCI container images.

Cargo.toml

@@ -32,24 +32,26 @@ rosenpass-secret-memory = { path = "secret-memory" }
 rosenpass-oqs = { path = "oqs" }
 rosenpass-wireguard-broker = { path = "wireguard-broker" }
 doc-comment = "0.3.3"
-base64ct = {version = "1.6.0", default-features=false}
+base64ct = { version = "1.6.0", default-features = false }
 zeroize = "1.8.1"
 memoffset = "0.9.1"
-thiserror = "1.0.61"
+thiserror = "1.0.64"
 paste = "1.0.15"
 env_logger = "0.10.2"
 toml = "0.7.8"
 static_assertions = "1.1.0"
 allocator-api2 = "0.2.14"
-memsec = { version="0.7.0", features = [ "alloc_ext", ] }
+memsec = { git = "https://github.com/rosenpass/memsec.git", rev = "aceb9baee8aec6844125bd6612f92e9a281373df", features = [
+    "alloc_ext",
+] }
 rand = "0.8.5"
 typenum = "1.17.0"
-log = { version = "0.4.21" }
-clap = { version = "4.5.7", features = ["derive"] }
-serde = { version = "1.0.203", features = ["derive"] }
+log = { version = "0.4.22" }
+clap = { version = "4.5.19", features = ["derive"] }
+serde = { version = "1.0.210", features = ["derive"] }
 arbitrary = { version = "1.3.2", features = ["derive"] }
-anyhow = { version = "1.0.86", features = ["backtrace", "std"] }
-mio = { version = "0.8.11", features = ["net", "os-poll"] }
+anyhow = { version = "1.0.89", features = ["backtrace", "std"] }
+mio = { version = "1.0.2", features = ["net", "os-poll"] }
 oqs-sys = { version = "0.9.1", default-features = false, features = [
     'classic_mceliece',
     'kyber',
@@ -59,24 +61,30 @@ chacha20poly1305 = { version = "0.10.1", default-features = false, features = [
     "std",
     "heapless",
 ] }
-zerocopy = { version = "0.7.34", features = ["derive"] }
+zerocopy = { version = "0.7.35", features = ["derive"] }
 home = "0.5.9"
-derive_builder = "0.20.0"
-tokio = { version = "1.38", features = ["macros", "rt-multi-thread"] }
-postcard= {version = "1.0.8", features = ["alloc"]}
+derive_builder = "0.20.1"
+tokio = { version = "1.40", features = ["macros", "rt-multi-thread"] }
+postcard = { version = "1.0.10", features = ["alloc"] }
+libcrux = { version = "0.0.2-pre.2" }
+hex-literal = { version = "0.4.1" }
+hex = { version = "0.4.3" }
+heck = { version = "0.5.0" }
+libc = { version = "0.2" }
+uds = { git = "https://github.com/rosenpass/uds" }
 
 #Dev dependencies
 serial_test = "3.1.1"
 tempfile = "3"
-stacker = "0.1.15"
+stacker = "0.1.17"
 libfuzzer-sys = "0.4"
 test_bin = "0.4.0"
 criterion = "0.4.0"
 allocator-api2-tests = "0.2.15"
-procspawn = {version = "1.0.0", features= ["test-support"]}
+procspawn = { version = "1.0.1", features = ["test-support"] }
 
 
 #Broker dependencies (might need cleanup or changes)
-wireguard-uapi = "3.0.0"
+wireguard-uapi = { version = "3.0.0", features = ["xplatform"] }
 command-fds = "0.2.3"
-rustix = { version = "0.38.27", features = ["net"] }
+rustix = { version = "0.38.37", features = ["net", "fs"] }

cipher-traits/src/kem.rs

@@ -10,8 +10,6 @@
 //! The [KEM] Trait describes the basic API offered by a Key Encapsulation
 //! Mechanism. Two implementations for it are provided, [StaticKEM] and [EphemeralKEM].
 
-use std::result::Result;
-
 /// Key Encapsulation Mechanism
 ///
 /// The KEM interface defines three operations: Key generation, key encapsulation and key

ciphers/Cargo.toml

@@ -9,6 +9,9 @@ homepage = "https://rosenpass.eu/"
 repository = "https://github.com/rosenpass/rosenpass"
 readme = "readme.md"
 
+[features]
+experiment_libcrux = ["dep:libcrux"]
+
 [dependencies]
 anyhow = { workspace = true }
 rosenpass-to = { workspace = true }
@@ -20,3 +23,4 @@ static_assertions = { workspace = true }
 zeroize = { workspace = true }
 chacha20poly1305 = { workspace = true }
 blake2 = { workspace = true }
+libcrux = { workspace = true, optional = true }

ciphers/src/lib.rs

@@ -9,7 +9,12 @@ const_assert!(KEY_LEN == hash_domain::KEY_LEN);
 
 /// Authenticated encryption with associated data
 pub mod aead {
+    #[cfg(not(feature = "experiment_libcrux"))]
     pub use crate::subtle::chacha20poly1305_ietf::{decrypt, encrypt, KEY_LEN, NONCE_LEN, TAG_LEN};
+    #[cfg(feature = "experiment_libcrux")]
+    pub use crate::subtle::chacha20poly1305_ietf_libcrux::{
+        decrypt, encrypt, KEY_LEN, NONCE_LEN, TAG_LEN,
+    };
 }
 
 /// Authenticated encryption with associated data with a constant nonce

ciphers/src/subtle/chacha20poly1305_ietf_libcrux.rs

@@ -0,0 +1,60 @@
+use rosenpass_to::ops::copy_slice;
+use rosenpass_to::To;
+
+use zeroize::Zeroize;
+
+pub const KEY_LEN: usize = 32; // Grrrr! Libcrux, please provide me these constants.
+pub const TAG_LEN: usize = 16;
+pub const NONCE_LEN: usize = 12;
+
+#[inline]
+pub fn encrypt(
+    ciphertext: &mut [u8],
+    key: &[u8],
+    nonce: &[u8],
+    ad: &[u8],
+    plaintext: &[u8],
+) -> anyhow::Result<()> {
+    let (ciphertext, mac) = ciphertext.split_at_mut(ciphertext.len() - TAG_LEN);
+
+    use libcrux::aead as C;
+    let crux_key = C::Key::Chacha20Poly1305(C::Chacha20Key(key.try_into().unwrap()));
+    let crux_iv = C::Iv(nonce.try_into().unwrap());
+
+    copy_slice(plaintext).to(ciphertext);
+    let crux_tag = libcrux::aead::encrypt(&crux_key, ciphertext, crux_iv, ad).unwrap();
+    copy_slice(crux_tag.as_ref()).to(mac);
+
+    match crux_key {
+        C::Key::Chacha20Poly1305(mut k) => k.0.zeroize(),
+        _ => panic!(),
+    }
+
+    Ok(())
+}
+
+#[inline]
+pub fn decrypt(
+    plaintext: &mut [u8],
+    key: &[u8],
+    nonce: &[u8],
+    ad: &[u8],
+    ciphertext: &[u8],
+) -> anyhow::Result<()> {
+    let (ciphertext, mac) = ciphertext.split_at(ciphertext.len() - TAG_LEN);
+
+    use libcrux::aead as C;
+    let crux_key = C::Key::Chacha20Poly1305(C::Chacha20Key(key.try_into().unwrap()));
+    let crux_iv = C::Iv(nonce.try_into().unwrap());
+    let crux_tag = C::Tag::from_slice(mac).unwrap();
+
+    copy_slice(ciphertext).to(plaintext);
+    libcrux::aead::decrypt(&crux_key, plaintext, crux_iv, ad, &crux_tag).unwrap();
+
+    match crux_key {
+        C::Key::Chacha20Poly1305(mut k) => k.0.zeroize(),
+        _ => panic!(),
+    }
+
+    Ok(())
+}

ciphers/src/subtle/mod.rs

@@ -1,4 +1,7 @@
 pub mod blake2b;
+#[cfg(not(feature = "experiment_libcrux"))]
 pub mod chacha20poly1305_ietf;
+#[cfg(feature = "experiment_libcrux")]
+pub mod chacha20poly1305_ietf_libcrux;
 pub mod incorrect_hmac_blake2b;
 pub mod xchacha20poly1305_ietf;

constant-time/src/memcmp.rs

@@ -7,18 +7,16 @@
 ///
 /// The execution time of the function grows approx. linear with the length of the input. This is
 /// considered safe.
-///
-/// ## Tests
-/// [`tests::memcmp_runs_in_constant_time`] runs a stasticial test that the equality of the two
-/// input parameters does not correlate with the run time.
-///
-/// For discussion on how to (further) ensure the constant-time execution of this function,
-/// see <https://github.com/rosenpass/rosenpass/issues/232>
 #[inline]
 pub fn memcmp(a: &[u8], b: &[u8]) -> bool {
     a.len() == b.len() && unsafe { memsec::memeq(a.as_ptr(), b.as_ptr(), a.len()) }
 }
 
+/// [tests::memcmp_runs_in_constant_time] runs a stasticial test that the equality of the two
+/// input parameters does not correlate with the run time.
+///
+/// For discussion on how to (further) ensure the constant-time execution of this function,
+/// see <https://github.com/rosenpass/rosenpass/issues/232>
 #[cfg(all(test, feature = "constant_time_tests"))]
 mod tests {
     use super::*;

doc/rosenpass.1

@@ -108,7 +108,7 @@ Rosenpass was created by Karolin Varner, Benjamin Lipp, Wanja Zaeske,
 Marei Peischl, Stephan Ajuvo, and Lisa Schmidt.
 .Pp
 This manual page was written by
-.An Emil Engler
+.An Clara Engler
 .Sh BUGS
 The bugs are tracked at
 .Lk https://github.com/rosenpass/rosenpass/issues .

doc/rp.1

@@ -113,7 +113,7 @@ Rosenpass was created by Karolin Varner, Benjamin Lipp, Wanja Zaeske,
 Marei Peischl, Stephan Ajuvo, and Lisa Schmidt.
 .Pp
 This manual page was written by
-.An Emil Engler
+.An Clara Engler
 .Sh BUGS
 The bugs are tracked at
 .Lk https://github.com/rosenpass/rosenpass/issues .

flake.lock

@@ -2,15 +2,17 @@
   "nodes": {
     "fenix": {
       "inputs": {
-        "nixpkgs": ["nixpkgs"],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1712298178,
-        "narHash": "sha256-590fpCPXYAkaAeBz/V91GX4/KGzPObdYtqsTWzT6AhI=",
+        "lastModified": 1728282832,
+        "narHash": "sha256-I7AbcwGggf+CHqpyd/9PiAjpIBGTGx5woYHqtwxaV7I=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "569b5b5781395da08e7064e825953c548c26af76",
+        "rev": "1ec71be1f4b8f3105c5d38da339cb061fefc43f4",
         "type": "github"
       },
       "original": {
@@ -24,11 +26,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -37,36 +39,18 @@
         "type": "github"
       }
     },
-    "naersk": {
-      "inputs": {
-        "nixpkgs": ["nixpkgs"]
-      },
-      "locked": {
-        "lastModified": 1698420672,
-        "narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "aeb58d5e8faead8980a807c840232697982d47b9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "naersk",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712168706,
-        "narHash": "sha256-XP24tOobf6GGElMd0ux90FEBalUtw6NkBSVh/RlA6ik=",
+        "lastModified": 1728193676,
+        "narHash": "sha256-PbDWAIjKJdlVg+qQRhzdSor04bAPApDqIv2DofTyynk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1487bdea619e4a7a53a4590c475deabb5a9d1bfb",
+        "rev": "ecbc1ca8ffd6aea8372ad16be9ebbb39889e55b6",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -75,18 +59,17 @@
       "inputs": {
         "fenix": "fenix",
         "flake-utils": "flake-utils",
-        "naersk": "naersk",
         "nixpkgs": "nixpkgs"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1712156296,
-        "narHash": "sha256-St7ZQrkrr5lmQX9wC1ZJAFxL8W7alswnyZk9d1se3Us=",
+        "lastModified": 1728249780,
+        "narHash": "sha256-J269DvCI5dzBmPrXhAAtj566qt0b22TJtF3TIK+tMsI=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "8e581ac348e223488622f4d3003cb2bd412bf27e",
+        "rev": "2b750da1a1a2c1d2c70896108d7096089842d877",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,12 +1,8 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
     flake-utils.url = "github:numtide/flake-utils";
 
-    # for quicker rust builds
-    naersk.url = "github:nix-community/naersk";
-    naersk.inputs.nixpkgs.follows = "nixpkgs";
-
     # for rust nightly with llvm-tools-preview
     fenix.url = "github:nix-community/fenix";
     fenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -15,6 +11,15 @@
   outputs = { self, nixpkgs, flake-utils, ... }@inputs:
     nixpkgs.lib.foldl (a: b: nixpkgs.lib.recursiveUpdate a b) { } [
 
+
+      #
+      ### Export the overlay.nix from this flake ###
+      #
+      {
+        overlays.default = import ./overlay.nix;
+      }
+
+
       #
       ### Actual Rosenpass Package and Docker Container Images ###
       #
@@ -30,310 +35,39 @@
       ]
         (system:
           let
-            scoped = (scope: scope.result);
-            lib = nixpkgs.lib;
-
             # normal nixpkgs
             pkgs = import nixpkgs {
               inherit system;
-            };
 
-            # parsed Cargo.toml
-            cargoToml = builtins.fromTOML (builtins.readFile ./rosenpass/Cargo.toml);
-
-            # source files relevant for rust
-            src = scoped rec {
-              # File suffices to include
-              extensions = [
-                "lock"
-                "rs"
-                "toml"
-              ];
-              # Files to explicitly include
-              files = [
-                "to/README.md"
-              ];
-
-              src = ./.;
-              filter = (path: type: scoped rec {
-                inherit (lib) any id removePrefix hasSuffix;
-                anyof = (any id);
-
-                basename = baseNameOf (toString path);
-                relative = removePrefix (toString src + "/") (toString path);
-
-                result = anyof [
-                  (type == "directory")
-                  (any (ext: hasSuffix ".${ext}" basename) extensions)
-                  (any (file: file == relative) files)
-                ];
-              });
-
-              result = pkgs.lib.sources.cleanSourceWith { inherit src filter; };
-            };
-
-            # a function to generate a nix derivation for rosenpass against any
-            # given set of nixpkgs
-            rosenpassDerivation = p:
-              let
-                # whether we want to build a statically linked binary
-                isStatic = p.targetPlatform.isStatic;
-
-                # the rust target of `p`
-                target = p.rust.toRustTargetSpec p.targetPlatform;
-
-                # convert a string to shout case
-                shout = string: builtins.replaceStrings [ "-" ] [ "_" ] (pkgs.lib.toUpper string);
-
-                # suitable Rust toolchain
-                toolchain = with inputs.fenix.packages.${system}; combine [
-                  stable.cargo
-                  stable.rustc
-                  targets.${target}.stable.rust-std
-                ];
-
-                # naersk with a custom toolchain
-                naersk = pkgs.callPackage inputs.naersk {
-                  cargo = toolchain;
-                  rustc = toolchain;
-                };
-
-                # used to trick the build.rs into believing that CMake was ran **again**
-                fakecmake = pkgs.writeScriptBin "cmake" ''
-                  #! ${pkgs.stdenv.shell} -e
-                  true
-                '';
-              in
-              naersk.buildPackage
-                {
-                  # metadata and source
-                  name = cargoToml.package.name;
-                  version = cargoToml.package.version;
-                  inherit src;
-
-                  cargoBuildOptions = x: x ++ [ "-p" "rosenpass" ];
-                  cargoTestOptions = x: x ++ [ "-p" "rosenpass" ];
-
-                  doCheck = true;
-
-                  nativeBuildInputs = with pkgs; [
-                    p.stdenv.cc
-                    cmake # for oqs build in the oqs-sys crate
-                    mandoc # for the built-in manual
-                    removeReferencesTo
-                    rustPlatform.bindgenHook # for C-bindings in the crypto libs
-                  ];
-                  buildInputs = with p; [ bash ];
-
-                  override = x: {
-                    preBuild =
-                      # nix defaults to building for aarch64 _without_ the armv8-a crypto
-                      # extensions, but liboqs depens on these
-                      (lib.optionalString (system == "aarch64-linux") ''
-                        NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -march=armv8-a+crypto"
-                      ''
-                      );
-
-                    # fortify is only compatible with dynamic linking
-                    hardeningDisable = lib.optional isStatic "fortify";
-                  };
-
-                  overrideMain = x: {
-                    # CMake detects that it was served a _foreign_ target dir, and CMake
-                    # would be executed again upon the second build step of naersk.
-                    # By adding our specially optimized CMake version, we reduce the cost
-                    # of recompilation by 99 % while, while avoiding any CMake errors.
-                    nativeBuildInputs = [ (lib.hiPrio fakecmake) ] ++ x.nativeBuildInputs;
-
-                    # make sure that libc is linked, under musl this is not the case per
-                    # default
-                    preBuild = (lib.optionalString isStatic ''
-                      NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -lc"
-                    '');
-                  };
-
-                  # We want to build for a specific target...
-                  CARGO_BUILD_TARGET = target;
-
-                  # ... which might require a non-default linker:
-                  "CARGO_TARGET_${shout target}_LINKER" =
-                    let
-                      inherit (p.stdenv) cc;
-                    in
-                    "${cc}/bin/${cc.targetPrefix}cc";
-
-                  meta = with pkgs.lib;
-                    {
-                      inherit (cargoToml.package) description homepage;
-                      license = with licenses; [ mit asl20 ];
-                      maintainers = [ maintainers.wucke13 ];
-                      platforms = platforms.all;
-                    };
-                } // (lib.mkIf isStatic {
-                # otherwise pkg-config tries to link non-existent dynamic libs
-                # documented here: https://docs.rs/pkg-config/latest/pkg_config/
-                PKG_CONFIG_ALL_STATIC = true;
-
-                # tell rust to build everything statically linked
-                CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static";
-              });
-            # a function to generate a nix derivation for the rp helper against any
-            # given set of nixpkgs
-            rpDerivation = p:
-              let
-                # whether we want to build a statically linked binary
-                isStatic = p.targetPlatform.isStatic;
-
-                # the rust target of `p`
-                target = p.rust.toRustTargetSpec p.targetPlatform;
-
-                # convert a string to shout case
-                shout = string: builtins.replaceStrings [ "-" ] [ "_" ] (pkgs.lib.toUpper string);
-
-                # suitable Rust toolchain
-                toolchain = with inputs.fenix.packages.${system}; combine [
-                  stable.cargo
-                  stable.rustc
-                  targets.${target}.stable.rust-std
-                ];
-
-                # naersk with a custom toolchain
-                naersk = pkgs.callPackage inputs.naersk {
-                  cargo = toolchain;
-                  rustc = toolchain;
-                };
-
-                # used to trick the build.rs into believing that CMake was ran **again**
-                fakecmake = pkgs.writeScriptBin "cmake" ''
-                  #! ${pkgs.stdenv.shell} -e
-                  true
-                '';
-              in
-              naersk.buildPackage
-                {
-                  # metadata and source
-                  name = cargoToml.package.name;
-                  version = cargoToml.package.version;
-                  inherit src;
-
-                  cargoBuildOptions = x: x ++ [ "-p" "rp" ];
-                  cargoTestOptions = x: x ++ [ "-p" "rp" ];
-
-                  doCheck = true;
-
-                  nativeBuildInputs = with pkgs; [
-                    p.stdenv.cc
-                    cmake # for oqs build in the oqs-sys crate
-                    mandoc # for the built-in manual
-                    removeReferencesTo
-                    rustPlatform.bindgenHook # for C-bindings in the crypto libs
-                  ];
-                  buildInputs = with p; [ bash ];
-
-                  override = x: {
-                    preBuild =
-                      # nix defaults to building for aarch64 _without_ the armv8-a crypto
-                      # extensions, but liboqs depens on these
-                      (lib.optionalString (system == "aarch64-linux") ''
-                        NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -march=armv8-a+crypto"
-                      ''
-                      );
-
-                    # fortify is only compatible with dynamic linking
-                    hardeningDisable = lib.optional isStatic "fortify";
-                  };
-
-                  overrideMain = x: {
-                    # CMake detects that it was served a _foreign_ target dir, and CMake
-                    # would be executed again upon the second build step of naersk.
-                    # By adding our specially optimized CMake version, we reduce the cost
-                    # of recompilation by 99 % while, while avoiding any CMake errors.
-                    nativeBuildInputs = [ (lib.hiPrio fakecmake) ] ++ x.nativeBuildInputs;
-
-                    # make sure that libc is linked, under musl this is not the case per
-                    # default
-                    preBuild = (lib.optionalString isStatic ''
-                      NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -lc"
-                    '');
-                  };
-
-                  # We want to build for a specific target...
-                  CARGO_BUILD_TARGET = target;
-
-                  # ... which might require a non-default linker:
-                  "CARGO_TARGET_${shout target}_LINKER" =
-                    let
-                      inherit (p.stdenv) cc;
-                    in
-                    "${cc}/bin/${cc.targetPrefix}cc";
-
-                  meta = with pkgs.lib;
-                    {
-                      inherit (cargoToml.package) description homepage;
-                      license = with licenses; [ mit asl20 ];
-                      maintainers = [ maintainers.wucke13 ];
-                      platforms = platforms.all;
-                    };
-                } // (lib.mkIf isStatic {
-                # otherwise pkg-config tries to link non-existent dynamic libs
-                # documented here: https://docs.rs/pkg-config/latest/pkg_config/
-                PKG_CONFIG_ALL_STATIC = true;
-
-                # tell rust to build everything statically linked
-                CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static";
-              });
-            # a function to generate a docker image based of rosenpass
-            rosenpassOCI = name: pkgs.dockerTools.buildImage rec {
-              inherit name;
-              copyToRoot = pkgs.buildEnv {
-                name = "image-root";
-                paths = [ self.packages.${system}.${name} ];
-                pathsToLink = [ "/bin" ];
-              };
-              config.Cmd = [ "/bin/rosenpass" ];
+              # apply our own overlay, overriding/inserting our packages as defined in ./pkgs
+              overlays = [ self.overlays.default ];
             };
           in
-          rec {
-            packages = rec {
-              default = rosenpass;
-              rosenpass = rosenpassDerivation pkgs;
-              rp = rpDerivation pkgs;
-              rosenpass-oci-image = rosenpassOCI "rosenpass";
+          {
+            packages = {
+              default = pkgs.rosenpass;
+              rosenpass = pkgs.rosenpass;
+              rosenpass-oci-image = pkgs.rosenpass-oci-image;
+              rp = pkgs.rp;
 
-              # derivation for the release
-              release-package =
-                let
-                  version = cargoToml.package.version;
-                  package =
-                    if pkgs.hostPlatform.isLinux then
-                      packages.rosenpass-static
-                    else packages.rosenpass;
-                  rp =
-                    if pkgs.hostPlatform.isLinux then
-                      packages.rp-static
-                    else packages.rp;
-                  oci-image =
-                    if pkgs.hostPlatform.isLinux then
-                      packages.rosenpass-static-oci-image
-                    else packages.rosenpass-oci-image;
-                in
-                pkgs.runCommandNoCC "lace-result" { }
-                  ''
-                    mkdir {bin,$out}
-                    tar -cvf $out/rosenpass-${system}-${version}.tar \
-                      -C ${package} bin/rosenpass \
-                      -C ${rp} bin/rp
-                    cp ${oci-image} \
-                      $out/rosenpass-oci-image-${system}-${version}.tar.gz
-                  '';
-            } // (if pkgs.stdenv.isLinux then rec {
-              rosenpass-static = rosenpassDerivation pkgs.pkgsStatic;
-              rp-static = rpDerivation pkgs.pkgsStatic;
-              rosenpass-static-oci-image = rosenpassOCI "rosenpass-static";
-            } else { });
+              release-package = pkgs.release-package;
+
+              # for good measure, we also offer to cross compile to Linux on Arm
+              aarch64-linux-rosenpass-static =
+                pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.rosenpass;
+              aarch64-linux-rp-static = pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.rp;
+            }
+            //
+            # We only offer static builds for linux, as this is not supported on OS X
+            (nixpkgs.lib.attrsets.optionalAttrs pkgs.stdenv.isLinux {
+              rosenpass-static = pkgs.pkgsStatic.rosenpass;
+              rosenpass-static-oci-image = pkgs.pkgsStatic.rosenpass-oci-image;
+              rp-static = pkgs.pkgsStatic.rp;
+            });
           }
         ))
 
+
       #
       ### Linux specifics ###
       #
@@ -341,88 +75,46 @@
         let
           pkgs = import nixpkgs {
             inherit system;
+
+            # apply our own overlay, overriding/inserting our packages as defined in ./pkgs
+            overlays = [ self.overlays.default ];
           };
-          packages = self.packages.${system};
         in
         {
-          #
-          ### Whitepaper ###
-          #
-          packages.whitepaper =
-            let
-              tlsetup = (pkgs.texlive.combine {
-                inherit (pkgs.texlive) scheme-basic acmart amsfonts ccicons
-                  csquotes csvsimple doclicense fancyvrb fontspec gobble
-                  koma-script ifmtarg latexmk lm markdown mathtools minted noto
-                  nunito pgf soul unicode-math lualatex-math paralist
-                  gitinfo2 eso-pic biblatex biblatex-trad biblatex-software
-                  xkeyval xurl xifthen biber;
-              });
-            in
-            pkgs.stdenvNoCC.mkDerivation {
-              name = "whitepaper";
-              src = ./papers;
-              nativeBuildInputs = with pkgs; [
-                ncurses # tput
-                python3Packages.pygments
-                tlsetup # custom tex live scheme
-                which
-              ];
-              buildPhase = ''
-                export HOME=$(mktemp -d)
-                latexmk -r tex/CI.rc
-              '';
-              installPhase = ''
-                mkdir -p $out
-                mv *.pdf readme.md $out/
-              '';
-            };
 
+          #
+          ### Reading materials ###
+          #
+          packages.whitepaper = pkgs.whitepaper;
 
           #
           ### Proof and Proof Tools ###
           #
-          packages.proverif-patched = pkgs.proverif.overrideAttrs (old: {
-            postInstall = ''
-              install -D -t $out/lib cryptoverif.pvl
-            '';
-          });
-          packages.proof-proverif = pkgs.stdenv.mkDerivation {
-            name = "rosenpass-proverif-proof";
-            version = "unstable";
-            src = pkgs.lib.sources.sourceByRegex ./. [
-              "analyze.sh"
-              "marzipan(/marzipan.awk)?"
-              "analysis(/.*)?"
-            ];
-            nativeBuildInputs = [ pkgs.proverif pkgs.graphviz ];
-            CRYPTOVERIF_LIB = packages.proverif-patched + "/lib/cryptoverif.pvl";
-            installPhase = ''
-              mkdir -p $out
-              bash analyze.sh -color -html $out
-            '';
-          };
+          packages.proverif-patched = pkgs.proverif-patched;
+          packages.proof-proverif = pkgs.proof-proverif;
 
 
           #
           ### Devshells ###
           #
           devShells.default = pkgs.mkShell {
-            inherit (packages.proof-proverif) CRYPTOVERIF_LIB;
-            inputsFrom = [ packages.default ];
+            inherit (pkgs.proof-proverif) CRYPTOVERIF_LIB;
+            inputsFrom = [ pkgs.rosenpass ];
             nativeBuildInputs = with pkgs; [
-              cmake # override the fakecmake from the main step above
               cargo-release
               clippy
+              rustfmt
               nodePackages.prettier
               nushell # for the .ci/gen-workflow-files.nu script
-              rustfmt
-              packages.proverif-patched
+              proverif-patched
             ];
           };
           devShells.coverage = pkgs.mkShell {
-            inputsFrom = [ packages.default ];
-            nativeBuildInputs = with pkgs; [ inputs.fenix.packages.${system}.complete.toolchain cargo-llvm-cov ];
+            inputsFrom = [ pkgs.rosenpass ];
+            nativeBuildInputs = [
+              inputs.fenix.packages.${system}.complete.toolchain
+              pkgs.cargo-llvm-cov
+            ];
           };
 
 

fuzz/Cargo.toml

@@ -4,6 +4,9 @@ version = "0.0.1"
 publish = false
 edition = "2021"
 
+[features]
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]
+
 [package.metadata]
 cargo-fuzz = true
 
@@ -81,4 +84,4 @@ doc = false
 name = "fuzz_vec_secret_alloc_memfdsec_mallocfb"
 path = "fuzz_targets/vec_secret_alloc_memfdsec_mallocfb.rs"
 test = false
-doc = false
+doc = false

fuzz/fuzz_targets/aead_enc_into.rs

@@ -15,8 +15,7 @@ pub struct Input {
 }
 
 fuzz_target!(|input: Input| {
-    let mut ciphertext: Vec<u8> = Vec::with_capacity(input.plaintext.len() + 16);
-    ciphertext.resize(input.plaintext.len() + 16, 0);
+    let mut ciphertext = vec![0u8; input.plaintext.len() + 16];
 
     aead::encrypt(
         ciphertext.as_mut_slice(),

fuzz/fuzz_targets/handle_msg.rs

@@ -7,14 +7,14 @@ use rosenpass::protocol::CryptoServer;
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
 use rosenpass_secret_memory::policy::*;
-use rosenpass_secret_memory::Secret;
+use rosenpass_secret_memory::{PublicBox, Secret};
 use std::sync::Once;
 
 static ONCE: Once = Once::new();
 fuzz_target!(|rx_buf: &[u8]| {
     ONCE.call_once(secret_policy_use_only_malloc_secrets);
     let sk = Secret::from_slice(&[0; StaticKem::SK_LEN]);
-    let pk = Secret::from_slice(&[0; StaticKem::PK_LEN]);
+    let pk = PublicBox::from_slice(&[0; StaticKem::PK_LEN]);
 
     let mut cs = CryptoServer::new(sk, pk);
     let mut tx_buf = [0; 10240];

fuzz/fuzz_targets/kyber_encaps.rs

@@ -14,7 +14,7 @@ pub struct Input {
 
 fuzz_target!(|input: Input| {
     let mut ciphertext = [0u8; EphemeralKem::CT_LEN];
-    let mut shared_secret = [0u8; EphemeralKem::SK_LEN];
+    let mut shared_secret = [0u8; EphemeralKem::SHK_LEN];
 
     EphemeralKem::encaps(&mut shared_secret, &mut ciphertext, &input.pk).unwrap();
 });

manual_tests/psk_broker/peer_a.rp.config

@@ -0,0 +1,13 @@
+secret_key = "peer_a.rp.sk"
+public_key = "peer_a.rp.pk"
+listen = ["[::1]:46127"]
+verbosity = "Verbose"
+
+[api]
+listen_path = []
+listen_fd = []
+stream_fd = []
+
+[[peers]]
+public_key = "peer_b.rp.pk"
+device = "rpPskBrkTestA"

manual_tests/psk_broker/peer_b.rp.config

@@ -0,0 +1,14 @@
+secret_key = "peer_b.rp.sk"
+public_key = "peer_b.rp.pk"
+listen = []
+verbosity = "Verbose"
+
+[api]
+listen_path = []
+listen_fd = []
+stream_fd = []
+
+[[peers]]
+public_key = "peer_a.rp.pk"
+endpoint = "[::1]:46127"
+device = "rpPskBrkTestB"

manual_tests/psk_broker/run_test.sh

@@ -0,0 +1,215 @@
+#! /bin/bash
+
+set -e -o pipefail
+
+enquote() {
+  while (( "$#" > 1)); do
+    printf "%q " "$1"
+    shift
+  done
+  if (("$#" > 0)); then
+    printf "%q" "$1"
+  fi
+}
+
+CLEANUP_HOOKS=()
+hook_cleanup() {
+  local hook
+  set +e +o pipefail
+  for hook in "${CLEANUP_HOOKS[@]}"; do
+    eval "${hook}"
+  done
+}
+
+cleanup() {
+  CLEANUP_HOOKS=("$(enquote exc_with_ctx cleanup "$@")" "${CLEANUP_HOOKS[@]}")
+}
+
+cleanup_eval() {
+  cleanup eval "$*"
+}
+
+stderr() {
+  echo >&2 "$@"
+}
+
+log() {
+  local level; level="$1"; shift || fatal "USAGE: log LVL MESSAGE.."
+  stderr "[${level}]" "$@"
+}
+
+info() {
+  log "INFO" "$@"
+}
+
+debug() {
+  log "DEBUG" "$@"
+}
+
+fatal() {
+  log "FATAL" "$@"
+  exit 1
+}
+
+assert() {
+  local msg; msg="$1"; shift || fatal "USAGE: assert_cmd MESSAGE COMMAND.."
+  "$@" || fatal "${msg}"
+}
+
+abs_dir() {
+  local dir; dir="$1"; shift || fatal "USAGE: abs_dir DIR"
+  (
+    cd "${dir}"
+    pwd -P
+  )
+}
+
+exc_with_ctx() {
+  local ctx; ctx="$1"; shift || fatal "USAGE: exc_with_ctx CONTEXT COMMAND.."
+  if [[ -z "${ctx}" ]]; then
+    info '$' "$@"
+  else
+    info "${ctx}\$" "$@"
+  fi
+
+  "$@"
+}
+
+exc() {
+  exc_with_ctx "" "$@"
+}
+
+exc_eval() {
+  exc eval "$*"
+} 
+
+exc_eval_with_ctx() {
+  local ctx; ctx="$1"; shift || fatal "USAGE: exc_eval_with_ctx CONTEXT EVAL_COMMAND.."
+  exc_with_ctx "eval:${ctx}" "$*"
+} 
+
+exc_as_user() {
+  exc sudo -u "${SUDO_USER}" "$@"
+}
+
+exc_eval_as_user() {
+  exc_as_user bash -c "$*"
+}
+
+fork_eval_as_user() {
+  exc sudo -u "${SUDO_USER}" bash -c "$*" &
+  local pid; pid="$!"
+  cleanup wait "${pid}"
+  cleanup pkill -2 -P "${pid}" # Reverse ordering
+}
+
+info_success() {
+  stderr
+  stderr
+  if [[ "${SUCCESS}" = 1 ]]; then
+    stderr "  Test was a success!"
+  else
+    stderr "  !!! TEST WAS A FAILURE!!!"
+  fi
+  stderr
+}
+
+main() {
+  assert "Use as root with sudo" [ "$(id -u)" -eq 0 ]
+  assert "Use as root with sudo" [ -n "${SUDO_UID}" ]
+  assert "SUDO_UID is 0; refusing to build as root" [ "${SUDO_UID}" -ne 0 ]
+
+  cleanup info_success
+
+  trap hook_cleanup EXIT
+
+  SCRIPT="$0"
+  CFG_TEMPLATE_DIR="$(abs_dir "$(dirname "${SCRIPT}")")"
+  REPO="$(abs_dir "${CFG_TEMPLATE_DIR}/../..")"
+  BINS="${REPO}/target/debug"
+
+  # Create temp dir
+  TMP_DIR="/tmp/rosenpass-psk-broker-test-$(date +%s)-$(uuidgen)"
+  cleanup rm -rf "${TMP_DIR}"
+  exc_as_user mkdir -p "${TMP_DIR}"
+
+  # Copy config
+  CFG_DIR="${TMP_DIR}/cfg"
+  exc_as_user cp -R "${CFG_TEMPLATE_DIR}" "${CFG_DIR}"
+
+  exc umask 077
+
+  exc cd "${REPO}"
+  local build_cmd; build_cmd=(cargo build --workspace --color=always --all-features --bins --profile dev)
+  if test -e "${BINS}/rosenpass-wireguard-broker-privileged" -a -e "${BINS}/rosenpass"; then
+    info "Found the binaries rosenpass-wireguard-broker-privileged and rosenpass." \
+      "Run following commands as a regular user to recompile the binaries with the right options" \
+      "in case of an error:" '$' "${build_cmd[@]}"
+  else
+    exc_as_user "${build_cmd[@]}"
+  fi
+  exc sudo setcap CAP_NET_ADMIN=+eip "${BINS}/rosenpass-wireguard-broker-privileged"
+
+  exc cd "${CFG_DIR}"
+  exc_eval_as_user "wg genkey > peer_a.wg.sk"
+  exc_eval_as_user "wg pubkey < peer_a.wg.sk > peer_a.wg.pk"
+  exc_eval_as_user "wg genkey > peer_b.wg.sk"
+  exc_eval_as_user "wg pubkey < peer_b.wg.sk > peer_b.wg.pk"
+  exc_eval_as_user "wg genpsk > peer_a_invalid.psk"
+  exc_eval_as_user "wg genpsk > peer_b_invalid.psk"
+  exc_eval_as_user "echo $(enquote "peer = \"$(cat peer_b.wg.pk)\"") >> peer_a.rp.config"
+  exc_eval_as_user "echo $(enquote "peer = \"$(cat peer_a.wg.pk)\"") >> peer_b.rp.config"
+  exc_as_user "${BINS}"/rosenpass gen-keys peer_a.rp.config
+  exc_as_user "${BINS}"/rosenpass gen-keys peer_b.rp.config
+
+  cleanup ip l del dev rpPskBrkTestA
+  cleanup ip l del dev rpPskBrkTestB
+  exc ip l add dev rpPskBrkTestA type wireguard
+  exc ip l add dev rpPskBrkTestB type wireguard
+
+  exc wg set rpPskBrkTestA \
+    listen-port 46125 \
+    private-key peer_a.wg.sk \
+    peer "$(cat peer_b.wg.pk)" \
+      endpoint 'localhost:46126' \
+      preshared-key peer_a_invalid.psk \
+      allowed-ips fe80::2/64
+  exc wg set rpPskBrkTestB \
+    listen-port 46126 \
+    private-key peer_b.wg.sk \
+    peer "$(cat peer_a.wg.pk)" \
+      endpoint 'localhost:46125' \
+      preshared-key peer_b_invalid.psk \
+      allowed-ips fe80::1/64
+
+  exc ip l set rpPskBrkTestA up
+  exc ip l set rpPskBrkTestB up
+
+  exc ip a add fe80::1/64 dev rpPskBrkTestA
+  exc ip a add fe80::2/64 dev rpPskBrkTestB
+
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "${REPO}/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass") --psk-broker-spawn \
+      exchange-config peer_a.rp.config"
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "${REPO}/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass-wireguard-broker-socket-handler") \
+      --listen-path broker.sock"
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "$PWD/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass") --psk-broker-path broker.sock \
+      exchange-config peer_b.rp.config"
+
+  exc_as_user ping -c 2 -w 10 fe80::1%rpPskBrkTestA
+  exc_as_user ping -c 2 -w 10 fe80::2%rpPskBrkTestB
+  exc_as_user ping -c 2 -w 10 fe80::2%rpPskBrkTestA
+  exc_as_user ping -c 2 -w 10 fe80::1%rpPskBrkTestB
+
+  SUCCESS=1
+}
+
+main "$@"

misc/README.md

@@ -0,0 +1,40 @@
+# Additional files
+
+This folder contains additional files that are used in the project.
+
+## `generate_configs.py`
+
+The script is used to generate configuration files for a benchmark setup
+consisting of a device under testing (DUT) and automatic test equipment (ATE),
+basically a strong machine capable of running multiple Rosenpass instances at
+once.
+
+At the top of the script multiple variables can be set to configure the DUT IP
+address and more. Once configured you may run `python3 generate_configs.py` to
+create the configuration files.
+
+A new folder called `output/` is created containing the subfolder `dut/` and
+`ate/`. The former has to be copied on the DUT, ideally reproducible hardware
+like a Raspberry Pi, while the latter is copied to the ATE, i.e. a laptop.
+
+### Running a benchmark
+
+On the ATE a run script is required since multiple instances of `rosenpass` are
+started with different configurations in parallel. The scripts are named after
+the number of instances they start, e.g. `run-50.sh` starts 50 instances.
+
+```shell
+# on the ATE aka laptop
+cd output/ate
+./run-10.sh
+```
+
+On the DUT you start a single Rosenpass instance with the configuration matching
+the ATE number of peers.
+
+```shell
+# on the DUT aka Raspberry Pi
+rosenpass exchange-config configs/dut-10.toml
+```
+
+Use whatever measurement tool you like to monitor the DUT and ATE.

misc/generate_configs.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+from subprocess import run
+import os
+
+config = dict(
+    peer_counts=[1, 5, 10, 50, 100, 500],
+    peer_count_max=100,
+    ate_ip="127.0.0.1",
+    dut_ip="127.0.0.1",
+    dut_port=9999,
+    path_to_rosenpass_bin=os.getcwd() + "/target/release/rosenpass",
+)
+
+print(config)
+
+output_dir = Path("output")
+output_dir.mkdir(exist_ok=True)
+
+template_dut = """
+public_key = "keys/dut-public-key"
+secret_key = "keys/dut-secret-key"
+listen = ["{dut_ip}:{dut_port}"]
+verbosity = "Quiet"
+"""
+template_dut_peer = """
+[[peers]] # ATE-{i}
+public_key = "keys/ate-{i}-public-key"
+endpoint = "{ate_ip}:{ate_port}"
+key_out = "out/key_out_{i}"
+"""
+
+template_ate = """
+public_key = "keys/ate-{i}-public-key"
+secret_key = "keys/ate-{i}-secret-key"
+listen = ["{ate_ip}:{ate_port}"]
+verbosity = "Quiet"
+
+[[peers]] # DUT
+public_key = "keys/dut-public-key"
+endpoint = "{dut_ip}:{dut_port}"
+key_out = "out/key_out_{i}"
+"""
+
+(output_dir / "dut" / "keys").mkdir(exist_ok=True, parents=True)
+(output_dir / "dut" / "out").mkdir(exist_ok=True, parents=True)
+(output_dir / "dut" / "configs").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "keys").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "out").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "configs").mkdir(exist_ok=True, parents=True)
+
+for peer_count in config["peer_counts"]:
+    dut_config = template_dut.format(**config)
+    for i in range(peer_count):
+        dut_config += template_dut_peer.format(**config, i=i, ate_port=50000 + i)
+
+    (output_dir / "dut" / "configs" / f"dut-{peer_count}.toml").write_text(dut_config)
+
+    if not (output_dir / "dut" / "keys" / "dut-public-key").exists():
+        print("Generate DUT keys")
+        run(
+            [
+                config["path_to_rosenpass_bin"],
+                "gen-keys",
+                f"configs/dut-{peer_count}.toml",
+            ],
+            cwd=output_dir / "dut",
+        )
+    else:
+        print("DUT keys already exist")
+
+# copy the DUT public key to the ATE
+(output_dir / "ate" / "keys" / "dut-public-key").write_bytes(
+    (output_dir / "dut" / "keys" / "dut-public-key").read_bytes()
+)
+
+ate_script = "(trap 'kill 0' SIGINT; \\\n"
+
+for i in range(config["peer_count_max"]):
+    (output_dir / "ate" / "configs" / f"ate-{i}.toml").write_text(
+        template_ate.format(**config, i=i, ate_port=50000 + i)
+    )
+
+    if not (output_dir / "ate" / "keys" / f"ate-{i}-public-key").exists():
+        # generate ATE keys
+        run(
+            [config["path_to_rosenpass_bin"], "gen-keys", f"configs/ate-{i}.toml"],
+            cwd=output_dir / "ate",
+        )
+    else:
+        print(f"ATE-{i} keys already exist")
+
+    # copy the ATE public keys to the DUT
+    (output_dir / "dut" / "keys" / f"ate-{i}-public-key").write_bytes(
+        (output_dir / "ate" / "keys" / f"ate-{i}-public-key").read_bytes()
+    )
+
+    ate_script += (
+        f"{config['path_to_rosenpass_bin']} exchange-config configs/ate-{i}.toml & \\\n"
+    )
+
+    if (i + 1) in config["peer_counts"]:
+        write_script = ate_script
+        write_script += "wait)"
+
+        (output_dir / "ate" / f"run-{i+1}.sh").write_text(write_script)

overlay.nix

@@ -0,0 +1,39 @@
+final: prev: {
+
+
+  #
+  ### Actual rosenpass software ###
+  #
+  rosenpass = final.callPackage ./pkgs/rosenpass.nix { };
+  rosenpass-oci-image = final.callPackage ./pkgs/rosenpass-oci-image.nix { };
+  rp = final.callPackage ./pkgs/rosenpass.nix { package = "rp"; };
+
+  release-package = final.callPackage ./pkgs/release-package.nix { };
+
+  #
+  ### Appendix ###
+  #
+  proverif-patched = prev.proverif.overrideAttrs (old: {
+    postInstall = ''
+      install -D -t $out/lib cryptoverif.pvl
+    '';
+  });
+
+  proof-proverif = final.stdenv.mkDerivation {
+    name = "rosenpass-proverif-proof";
+    version = "unstable";
+    src = final.lib.sources.sourceByRegex ./. [
+      "analyze.sh"
+      "marzipan(/marzipan.awk)?"
+      "analysis(/.*)?"
+    ];
+    nativeBuildInputs = [ final.proverif final.graphviz ];
+    CRYPTOVERIF_LIB = final.proverif-patched + "/lib/cryptoverif.pvl";
+    installPhase = ''
+      mkdir -p $out
+      bash analyze.sh -color -html $out
+    '';
+  };
+
+  whitepaper = final.callPackage ./pkgs/whitepaper.nix { };
+}

pkgs/release-package.nix

@@ -0,0 +1,27 @@
+{ lib, stdenvNoCC, runCommandNoCC, pkgsStatic, rosenpass, rosenpass-oci-image, rp } @ args:
+
+let
+  version = rosenpass.version;
+
+  # select static packages on Linux, default packages otherwise
+  package =
+    if stdenvNoCC.hostPlatform.isLinux then
+      pkgsStatic.rosenpass
+    else args.rosenpass;
+  rp =
+    if stdenvNoCC.hostPlatform.isLinux then
+      pkgsStatic.rp
+    else args.rp;
+  oci-image =
+    if stdenvNoCC.hostPlatform.isLinux then
+      pkgsStatic.rosenpass-oci-image
+    else args.rosenpass-oci-image;
+in
+runCommandNoCC "lace-result" { } ''
+  mkdir {bin,$out}
+  tar -cvf $out/rosenpass-${stdenvNoCC.hostPlatform.system}-${version}.tar \
+    -C ${package} bin/rosenpass \
+    -C ${rp} bin/rp
+  cp ${oci-image} \
+    $out/rosenpass-oci-image-${stdenvNoCC.hostPlatform.system}-${version}.tar.gz
+''

pkgs/rosenpass-oci-image.nix

@@ -0,0 +1,11 @@
+{ dockerTools, buildEnv, rosenpass }:
+
+dockerTools.buildImage {
+  name = rosenpass.name + "-oci";
+  copyToRoot = buildEnv {
+    name = "image-root";
+    paths = [ rosenpass ];
+    pathsToLink = [ "/bin" ];
+  };
+  config.Cmd = [ "/bin/rosenpass" ];
+}

pkgs/rosenpass.nix

@@ -0,0 +1,78 @@
+{ lib, stdenv, rustPlatform, cmake, mandoc, removeReferencesTo, bash, package ? "rosenpass" }:
+
+let
+  # whether we want to build a statically linked binary
+  isStatic = stdenv.targetPlatform.isStatic;
+
+  scoped = (scope: scope.result);
+
+  # source files relevant for rust
+  src = scoped rec {
+    # File suffices to include
+    extensions = [
+      "lock"
+      "rs"
+      "toml"
+    ];
+    # Files to explicitly include
+    files = [
+      "to/README.md"
+    ];
+
+    src = ../.;
+    filter = (path: type: scoped rec {
+      inherit (lib) any id removePrefix hasSuffix;
+      anyof = (any id);
+
+      basename = baseNameOf (toString path);
+      relative = removePrefix (toString src + "/") (toString path);
+
+      result = anyof [
+        (type == "directory")
+        (any (ext: hasSuffix ".${ext}" basename) extensions)
+        (any (file: file == relative) files)
+      ];
+    });
+
+    result = lib.sources.cleanSourceWith { inherit src filter; };
+  };
+
+  # parsed Cargo.toml
+  cargoToml = builtins.fromTOML (builtins.readFile (src + "/rosenpass/Cargo.toml"));
+in
+rustPlatform.buildRustPackage {
+  name = cargoToml.package.name;
+  version = cargoToml.package.version;
+  inherit src;
+
+  cargoBuildOptions = [ "--package" package ];
+  cargoTestOptions = [ "--package" package ];
+
+  doCheck = true;
+
+  cargoLock = {
+    lockFile = src + "/Cargo.lock";
+    outputHashes = {
+      "memsec-0.6.3" = "sha256-4ri+IEqLd77cLcul3lZrmpDKj4cwuYJ8oPRAiQNGeLw=";
+      "uds-0.4.2" = "sha256-qlxr/iJt2AV4WryePIvqm/8/MK/iqtzegztNliR93W8=";
+    };
+  };
+
+  nativeBuildInputs = [
+    stdenv.cc
+    cmake # for oqs build in the oqs-sys crate
+    mandoc # for the built-in manual
+    removeReferencesTo
+    rustPlatform.bindgenHook # for C-bindings in the crypto libs
+  ];
+  buildInputs = [ bash ];
+
+  hardeningDisable = lib.optional isStatic "fortify";
+
+  meta = {
+    inherit (cargoToml.package) description homepage;
+    license = with lib.licenses; [ mit asl20 ];
+    maintainers = [ lib.maintainers.wucke13 ];
+    platforms = lib.platforms.all;
+  };
+}

pkgs/whitepaper.nix

@@ -0,0 +1,29 @@
+{ stdenvNoCC, texlive, ncurses, python3Packages, which }:
+
+let
+  customTexLiveSetup = (texlive.combine {
+    inherit (texlive) acmart amsfonts biber biblatex biblatex-software
+      biblatex-trad ccicons csquotes csvsimple doclicense eso-pic fancyvrb
+      fontspec gitinfo2 gobble ifmtarg koma-script latexmk lm lualatex-math
+      markdown mathtools minted noto nunito paralist pgf scheme-basic soul
+      unicode-math upquote xifthen xkeyval xurl;
+  });
+in
+stdenvNoCC.mkDerivation {
+  name = "whitepaper";
+  src = ../papers;
+  nativeBuildInputs = [
+    ncurses # tput
+    python3Packages.pygments
+    customTexLiveSetup # custom tex live scheme
+    which
+  ];
+  buildPhase = ''
+    export HOME=$(mktemp -d)
+    latexmk -r tex/CI.rc
+  '';
+  installPhase = ''
+    mkdir -p $out
+    mv *.pdf readme.md $out/
+  '';
+}

readme.md

@@ -66,6 +66,8 @@ A wrapper script provides instant feedback about which queries execute as expect
 
 # Getting Rosenpass
 
+Documentation and installation guides can be found at the [Rosenpass website](https://rosenpass.eu/docs).
+
 Rosenpass is packaged for more and more distributions, maybe also for the distribution of your choice?
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/rosenpass.svg)](https://repology.org/project/rosenpass/versions)

rosenpass/Cargo.toml

@@ -13,6 +13,19 @@ readme = "readme.md"
 name = "rosenpass"
 path = "src/main.rs"
 
+[[bin]]
+name = "rosenpass-gen-ipc-msg-types"
+path = "src/bin/gen-ipc-msg-types.rs"
+required-features = ["experiment_api", "internal_bin_gen_ipc_msg_types"]
+
+[[test]]
+name = "api-integration-tests"
+required-features = ["experiment_api", "internal_testing"]
+
+[[test]]
+name = "api-integration-tests-api-setup"
+required-features = ["experiment_api", "internal_testing"]
+
 [[bench]]
 name = "handshake"
 harness = false
@@ -38,8 +51,15 @@ mio = { workspace = true }
 rand = { workspace = true }
 zerocopy = { workspace = true }
 home = { workspace = true }
-derive_builder = {workspace = true}
-rosenpass-wireguard-broker = {workspace = true}
+derive_builder = { workspace = true }
+rosenpass-wireguard-broker = { workspace = true }
+zeroize = { workspace = true }
+hex-literal = { workspace = true, optional = true }
+hex = { workspace = true, optional = true }
+heck = { workspace = true, optional = true }
+command-fds = { workspace = true, optional = true }
+rustix = { workspace = true }
+uds = { workspace = true, optional = true, features = ["mio_1xx"] }
 
 [build-dependencies]
 anyhow = { workspace = true }
@@ -48,9 +68,21 @@ anyhow = { workspace = true }
 criterion = { workspace = true }
 test_bin = { workspace = true }
 stacker = { workspace = true }
-serial_test = {workspace = true}
-procspawn = {workspace = true}
+serial_test = { workspace = true }
+procspawn = { workspace = true }
+tempfile = { workspace = true }
+rustix = { workspace = true }
 
 [features]
-enable_broker_api = ["rosenpass-wireguard-broker/enable_broker_api"]
-enable_memfd_alloc = []
+default = ["experiment_api"]
+experiment_memfd_secret = ["rosenpass-wireguard-broker/experiment_memfd_secret"]
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]
+experiment_api = [
+    "hex-literal",
+    "uds",
+    "command-fds",
+    "rosenpass-util/experiment_file_descriptor_passing",
+    "rosenpass-wireguard-broker/experiment_api",
+]
+internal_testing = []
+internal_bin_gen_ipc_msg_types = ["hex", "heck"]

rosenpass/benches/handshake.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 use rosenpass::protocol::{CryptoServer, HandleMsgResult, MsgBuf, PeerPtr, SPk, SSk, SymKey};
+use std::ops::DerefMut;
 
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
@@ -40,7 +41,7 @@ fn hs(ini: &mut CryptoServer, res: &mut CryptoServer) -> Result<()> {
 
 fn keygen() -> Result<(SSk, SPk)> {
     let (mut sk, mut pk) = (SSk::zero(), SPk::zero());
-    StaticKem::keygen(sk.secret_mut(), pk.secret_mut())?;
+    StaticKem::keygen(sk.secret_mut(), pk.deref_mut())?;
     Ok((sk, pk))
 }
 

rosenpass/src/api/api_handler.rs

@@ -0,0 +1,295 @@
+use std::{borrow::BorrowMut, collections::VecDeque, os::fd::OwnedFd};
+
+use anyhow::Context;
+use rosenpass_to::{ops::copy_slice, To};
+use rosenpass_util::{
+    fd::FdIo,
+    functional::{run, ApplyExt},
+    io::ReadExt,
+    mem::DiscardResultExt,
+    mio::UnixStreamExt,
+    result::OkExt,
+};
+use rosenpass_wireguard_broker::brokers::mio_client::MioBrokerClient;
+
+use crate::{
+    api::{add_listen_socket_response_status, add_psk_broker_response_status},
+    app_server::AppServer,
+    protocol::BuildCryptoServer,
+};
+
+use super::{supply_keypair_response_status, Server as ApiServer};
+
+#[derive(Debug)]
+pub struct ApiHandler {
+    _dummy: (),
+}
+
+impl ApiHandler {
+    #[allow(clippy::new_without_default)]
+    pub fn new() -> Self {
+        Self { _dummy: () }
+    }
+}
+
+pub trait ApiHandlerContext {
+    fn api_handler(&self) -> &ApiHandler;
+    fn app_server(&self) -> &AppServer;
+    fn api_handler_mut(&mut self) -> &mut ApiHandler;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+}
+
+#[derive(thiserror::Error, Debug)]
+#[error("Error in SupplyKeypair")]
+struct SupplyKeypairError {
+    status: u128,
+    #[source]
+    cause: anyhow::Error,
+}
+
+trait SupplyKeypairErrorExt<T> {
+    fn e_custom(self, status: u128) -> Result<T, SupplyKeypairError>;
+    fn einternal(self) -> Result<T, SupplyKeypairError>;
+    fn ealready_supplied(self) -> Result<T, SupplyKeypairError>;
+    fn einvalid_req(self) -> Result<T, SupplyKeypairError>;
+}
+
+impl<T, E: Into<anyhow::Error>> SupplyKeypairErrorExt<T> for Result<T, E> {
+    fn e_custom(self, status: u128) -> Result<T, SupplyKeypairError> {
+        self.map_err(|e| SupplyKeypairError {
+            status,
+            cause: e.into(),
+        })
+    }
+
+    fn einternal(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::INTERNAL_ERROR)
+    }
+
+    fn ealready_supplied(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::KEYPAIR_ALREADY_SUPPLIED)
+    }
+
+    fn einvalid_req(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::INVALID_REQUEST)
+    }
+}
+
+impl<T> ApiServer for T
+where
+    T: ?Sized + ApiHandlerContext,
+{
+    fn ping(
+        &mut self,
+        req: &super::PingRequest,
+        _req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::PingResponse,
+    ) -> anyhow::Result<()> {
+        let (req, res) = (&req.payload, &mut res.payload);
+        copy_slice(&req.echo).to(&mut res.echo);
+        Ok(())
+    }
+
+    fn supply_keypair(
+        &mut self,
+        req: &super::SupplyKeypairRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::SupplyKeypairResponse,
+    ) -> anyhow::Result<()> {
+        let outcome: Result<(), SupplyKeypairError> = run(|| {
+            // Acquire the file descriptors
+            let mut sk_io = FdIo(
+                req_fds
+                    .front()
+                    .context("First file descriptor, secret key, missing.")
+                    .einvalid_req()?,
+            );
+            let mut pk_io = FdIo(
+                req_fds
+                    .get(1)
+                    .context("Second file descriptor, public key, missing.")
+                    .einvalid_req()?,
+            );
+
+            // Actually read the secrets
+            let mut sk = crate::protocol::SSk::zero();
+            sk_io.read_exact_til_end(sk.secret_mut()).einvalid_req()?;
+
+            let mut pk = crate::protocol::SPk::zero();
+            pk_io.read_exact_til_end(pk.borrow_mut()).einvalid_req()?;
+
+            // Retrieve the construction site
+            let construction_site = self.app_server_mut().crypto_site.borrow_mut();
+
+            // Retrieve the builder
+            use rosenpass_util::build::ConstructionSite as C;
+            let maybe_builder = match construction_site {
+                C::Builder(builder) => Some(builder),
+                C::Product(_) => None,
+                C::Void => {
+                    return Err(anyhow::Error::msg("CryptoServer construction side is void"))
+                        .einternal();
+                }
+            };
+
+            // Retrieve a reference to the keypair
+            let Some(BuildCryptoServer {
+                ref mut keypair, ..
+            }) = maybe_builder
+            else {
+                return Err(anyhow::Error::msg("CryptoServer already built")).ealready_supplied();
+            };
+
+            // Supply the keypair to the CryptoServer
+            keypair
+                .insert(crate::protocol::Keypair { sk, pk })
+                .discard_result();
+
+            // Actually construct the CryptoServer
+            construction_site
+                .erect()
+                .map_err(|e| anyhow::Error::msg(format!("Error erecting the CryptoServer {e:?}")))
+                .einternal()?;
+
+            Ok(())
+        });
+
+        // Handle errors
+        use supply_keypair_response_status as status;
+        let status = match outcome {
+            Ok(()) => status::OK,
+            Err(e) => {
+                let lvl = match e.status {
+                    status::INTERNAL_ERROR => log::Level::Warn,
+                    _ => log::Level::Debug,
+                };
+
+                log::log!(
+                    lvl,
+                    "Error while processing API Request.\n    Request: {:?}\n    Error: {:?}",
+                    req,
+                    e.cause
+                );
+
+                if e.status == status::INTERNAL_ERROR {
+                    return Err(e.cause);
+                }
+
+                e.status
+            }
+        };
+
+        res.payload.status = status;
+
+        Ok(())
+    }
+
+    fn add_listen_socket(
+        &mut self,
+        _req: &super::boilerplate::AddListenSocketRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::boilerplate::AddListenSocketResponse,
+    ) -> anyhow::Result<()> {
+        // Retrieve file descriptor
+        let sock_res = run(|| -> anyhow::Result<mio::net::UdpSocket> {
+            let sock = req_fds
+                .pop_front()
+                .context("Invalid request – socket missing.")?;
+            // TODO: We need to have this outside linux
+            #[cfg(target_os = "linux")]
+            rosenpass_util::fd::GetSocketProtocol::demand_udp_socket(&sock)?;
+            let sock = std::net::UdpSocket::from(sock);
+            sock.set_nonblocking(true)?;
+            mio::net::UdpSocket::from_std(sock).ok()
+        });
+
+        let sock = match sock_res {
+            Ok(sock) => sock,
+            Err(e) => {
+                log::debug!("Error processing AddListenSocket API request: {e:?}");
+                res.payload.status = add_listen_socket_response_status::INVALID_REQUEST;
+                return Ok(());
+            }
+        };
+
+        // Register socket
+        let reg_result = self.app_server_mut().register_listen_socket(sock);
+
+        if let Err(internal_error) = reg_result {
+            log::warn!("Internal error processing AddListenSocket API request: {internal_error:?}");
+            res.payload.status = add_listen_socket_response_status::INTERNAL_ERROR;
+            return Ok(());
+        };
+
+        res.payload.status = add_listen_socket_response_status::OK;
+        Ok(())
+    }
+
+    fn add_psk_broker(
+        &mut self,
+        _req: &super::boilerplate::AddPskBrokerRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::boilerplate::AddPskBrokerResponse,
+    ) -> anyhow::Result<()> {
+        // Retrieve file descriptor
+        let sock_res = run(|| {
+            let sock = req_fds
+                .pop_front()
+                .context("Invalid request – socket missing.")?;
+            mio::net::UnixStream::from_fd(sock)
+        });
+
+        // Handle errors
+        let sock = match sock_res {
+            Ok(sock) => sock,
+            Err(e) => {
+                log::debug!(
+                    "Request found to be invalid while processing AddPskBroker API request: {e:?}"
+                );
+                res.payload.status = add_psk_broker_response_status::INVALID_REQUEST;
+                return Ok(());
+            }
+        };
+
+        // Register Socket
+        let client = Box::new(MioBrokerClient::new(sock));
+
+        // Workaround: The broker code is currently impressively overcomplicated. Brokers are
+        // stored in a hash map but the hash map key used is just a counter so a vector could
+        // have been used. Broker configuration is abstracted, different peers can have different
+        // brokers but there is no facility to add multiple brokers in practice. The broker index
+        // uses a `Public` wrapper without actually holding any cryptographic data. Even the broker
+        // configuration uses a trait abstraction for no discernible reason and a lot of the code
+        // introduces pointless, single-field wrapper structs.
+        // We should use an implement-what-is-actually-needed strategy next time.
+        // The Broker code needs to be slimmed down, the right direction to go is probably to
+        // just add event and capability support to the API and use the API to deliver OSK events.
+        //
+        // For now, we just replace the latest broker.
+        let erase_ptr = {
+            use crate::app_server::BrokerStorePtr;
+            //
+            use rosenpass_secret_memory::Public;
+            use zerocopy::AsBytes;
+            (self.app_server().brokers.store.len() - 1)
+                .apply(|x| x as u64)
+                .apply(|x| Public::from_slice(x.as_bytes()))
+                .apply(BrokerStorePtr)
+        };
+
+        let register_result = run(|| {
+            let srv = self.app_server_mut();
+            srv.unregister_broker(erase_ptr)?;
+            srv.register_broker(client)
+        });
+
+        if let Err(e) = register_result {
+            log::warn!("Internal error while processing AddPskBroker API request: {e:?}");
+            res.payload.status = add_psk_broker_response_status::INTERNAL_ERROR;
+            return Ok(());
+        }
+
+        res.payload.status = add_psk_broker_response_status::OK;
+        Ok(())
+    }
+}

rosenpass/src/api/boilerplate/byte_slice_ext.rs

@@ -0,0 +1,222 @@
+use zerocopy::{ByteSlice, Ref};
+
+use rosenpass_util::zerocopy::{RefMaker, ZerocopySliceExt};
+
+use super::{
+    PingRequest, PingResponse, RawMsgType, RefMakerRawMsgTypeExt, RequestMsgType, RequestRef,
+    ResponseMsgType, ResponseRef, SupplyKeypairRequest, SupplyKeypairResponse,
+};
+
+pub trait ByteSliceRefExt: ByteSlice {
+    fn msg_type_maker(self) -> RefMaker<Self, RawMsgType> {
+        self.zk_ref_maker()
+    }
+
+    fn msg_type(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse()
+    }
+
+    fn msg_type_from_prefix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn msg_type_from_suffix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn request_msg_type(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker().parse_request_msg_type()
+    }
+
+    fn request_msg_type_from_prefix(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker()
+            .from_prefix()?
+            .parse_request_msg_type()
+    }
+
+    fn request_msg_type_from_suffix(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker()
+            .from_suffix()?
+            .parse_request_msg_type()
+    }
+
+    fn response_msg_type(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker().parse_response_msg_type()
+    }
+
+    fn response_msg_type_from_prefix(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker()
+            .from_prefix()?
+            .parse_response_msg_type()
+    }
+
+    fn response_msg_type_from_suffix(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker()
+            .from_suffix()?
+            .parse_response_msg_type()
+    }
+
+    fn parse_request(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse(self)
+    }
+
+    fn parse_request_from_prefix(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse_from_prefix(self)
+    }
+
+    fn parse_request_from_suffix(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse_from_suffix(self)
+    }
+
+    fn parse_response(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse(self)
+    }
+
+    fn parse_response_from_prefix(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse_from_prefix(self)
+    }
+
+    fn parse_response_from_suffix(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse_from_suffix(self)
+    }
+
+    fn ping_request_maker(self) -> RefMaker<Self, PingRequest> {
+        self.zk_ref_maker()
+    }
+
+    fn ping_request(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse()
+    }
+
+    fn ping_request_from_prefix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn ping_request_from_suffix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn ping_response_maker(self) -> RefMaker<Self, PingResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn ping_response(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse()
+    }
+
+    fn ping_response_from_prefix(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn ping_response_from_suffix(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn supply_keypair_request(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse()
+    }
+
+    fn supply_keypair_request_from_prefix(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn supply_keypair_request_from_suffix(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn supply_keypair_response_maker(self) -> RefMaker<Self, SupplyKeypairResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn supply_keypair_response(self) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse()
+    }
+
+    fn supply_keypair_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn supply_keypair_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_listen_socket_request(self) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse()
+    }
+
+    fn add_listen_socket_request_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_listen_socket_request_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_listen_socket_response_maker(self) -> RefMaker<Self, super::AddListenSocketResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn add_listen_socket_response(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse()
+    }
+
+    fn add_listen_socket_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_listen_socket_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_psk_broker_request(self) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse()
+    }
+
+    fn add_psk_broker_request_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_psk_broker_request_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_psk_broker_response_maker(self) -> RefMaker<Self, super::AddPskBrokerResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn add_psk_broker_response(self) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse()
+    }
+
+    fn add_psk_broker_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_psk_broker_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse_suffix()
+    }
+}
+
+impl<B: ByteSlice> ByteSliceRefExt for B {}

rosenpass/src/api/boilerplate/message_trait.rs

@@ -0,0 +1,29 @@
+use zerocopy::{ByteSliceMut, Ref};
+
+use rosenpass_util::zerocopy::RefMaker;
+
+use super::RawMsgType;
+
+pub trait Message {
+    type Payload;
+    type MessageClass: Into<RawMsgType>;
+    const MESSAGE_TYPE: Self::MessageClass;
+
+    fn from_payload(payload: Self::Payload) -> Self;
+    fn init(&mut self);
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>>;
+}
+
+pub trait ZerocopyResponseMakerSetupMessageExt<B, T> {
+    fn setup_msg(self) -> anyhow::Result<Ref<B, T>>;
+}
+
+impl<B, T> ZerocopyResponseMakerSetupMessageExt<B, T> for RefMaker<B, T>
+where
+    B: ByteSliceMut,
+    T: Message,
+{
+    fn setup_msg(self) -> anyhow::Result<Ref<B, T>> {
+        T::setup(self.into_buf())
+    }
+}

rosenpass/src/api/boilerplate/message_type.rs

@@ -0,0 +1,162 @@
+use hex_literal::hex;
+use rosenpass_util::zerocopy::RefMaker;
+use zerocopy::ByteSlice;
+
+use crate::RosenpassError::{self, InvalidApiMessageType};
+
+pub type RawMsgType = u128;
+
+// constants generated by gen-ipc-msg-types:
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Ping Request
+pub const PING_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("2397 3ecc c441 704d    0b02 ea31 45d3 4999"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Ping Response
+pub const PING_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("4ec7 f6f0 2bbc ba64    48f1 da14 c7cf 0260"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Supply Keypair Request
+const SUPPLY_KEYPAIR_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("ac91 a5a6 4f4b 21d0    ac7f 9b55 74f7 3529"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Supply Keypair Response
+const SUPPLY_KEYPAIR_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("f2dc 49bd e261 5f10    40b7 3c16 ec61 edb9"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Listen Socket Request
+const ADD_LISTEN_SOCKET_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("3f21 434f 87cc a08c    02c4 61e4 0816 c7da"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Listen Socket Response
+const ADD_LISTEN_SOCKET_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("45d5 0f0d 93f0 6105    98f2 9469 5dfd 5f36"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Psk Broker Request
+const ADD_PSK_BROKER_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("d798 b8dc bd61 5cab    8df1 c63d e4eb a2d1"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Psk Broker Response
+const ADD_PSK_BROKER_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("bd25 e418 ffb0 6930    248b 217e 2fae e353"));
+
+pub trait MessageAttributes {
+    fn message_size(&self) -> usize;
+}
+
+#[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+pub enum RequestMsgType {
+    Ping,
+    SupplyKeypair,
+    AddListenSocket,
+    AddPskBroker,
+}
+
+#[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+pub enum ResponseMsgType {
+    Ping,
+    SupplyKeypair,
+    AddListenSocket,
+    AddPskBroker,
+}
+
+impl MessageAttributes for RequestMsgType {
+    fn message_size(&self) -> usize {
+        match self {
+            Self::Ping => std::mem::size_of::<super::PingRequest>(),
+            Self::SupplyKeypair => std::mem::size_of::<super::SupplyKeypairRequest>(),
+            Self::AddListenSocket => std::mem::size_of::<super::AddListenSocketRequest>(),
+            Self::AddPskBroker => std::mem::size_of::<super::AddPskBrokerRequest>(),
+        }
+    }
+}
+
+impl MessageAttributes for ResponseMsgType {
+    fn message_size(&self) -> usize {
+        match self {
+            Self::Ping => std::mem::size_of::<super::PingResponse>(),
+            Self::SupplyKeypair => std::mem::size_of::<super::SupplyKeypairResponse>(),
+            Self::AddListenSocket => std::mem::size_of::<super::AddListenSocketResponse>(),
+            Self::AddPskBroker => std::mem::size_of::<super::AddPskBrokerResponse>(),
+        }
+    }
+}
+
+impl TryFrom<RawMsgType> for RequestMsgType {
+    type Error = RosenpassError;
+
+    fn try_from(value: RawMsgType) -> Result<Self, Self::Error> {
+        use RequestMsgType as E;
+        Ok(match value {
+            self::PING_REQUEST => E::Ping,
+            self::SUPPLY_KEYPAIR_REQUEST => E::SupplyKeypair,
+            self::ADD_LISTEN_SOCKET_REQUEST => E::AddListenSocket,
+            self::ADD_PSK_BROKER_REQUEST => E::AddPskBroker,
+            _ => return Err(InvalidApiMessageType(value)),
+        })
+    }
+}
+
+impl From<RequestMsgType> for RawMsgType {
+    fn from(val: RequestMsgType) -> Self {
+        use RequestMsgType as E;
+        match val {
+            E::Ping => self::PING_REQUEST,
+            E::SupplyKeypair => self::SUPPLY_KEYPAIR_REQUEST,
+            E::AddListenSocket => self::ADD_LISTEN_SOCKET_REQUEST,
+            E::AddPskBroker => self::ADD_PSK_BROKER_REQUEST,
+        }
+    }
+}
+
+impl TryFrom<RawMsgType> for ResponseMsgType {
+    type Error = RosenpassError;
+
+    fn try_from(value: RawMsgType) -> Result<Self, Self::Error> {
+        use ResponseMsgType as E;
+        Ok(match value {
+            self::PING_RESPONSE => E::Ping,
+            self::SUPPLY_KEYPAIR_RESPONSE => E::SupplyKeypair,
+            self::ADD_LISTEN_SOCKET_RESPONSE => E::AddListenSocket,
+            self::ADD_PSK_BROKER_RESPONSE => E::AddPskBroker,
+            _ => return Err(InvalidApiMessageType(value)),
+        })
+    }
+}
+
+impl From<ResponseMsgType> for RawMsgType {
+    fn from(val: ResponseMsgType) -> Self {
+        use ResponseMsgType as E;
+        match val {
+            E::Ping => self::PING_RESPONSE,
+            E::SupplyKeypair => self::SUPPLY_KEYPAIR_RESPONSE,
+            E::AddListenSocket => self::ADD_LISTEN_SOCKET_RESPONSE,
+            E::AddPskBroker => self::ADD_PSK_BROKER_RESPONSE,
+        }
+    }
+}
+
+pub trait RawMsgTypeExt {
+    fn into_request_msg_type(self) -> Result<RequestMsgType, RosenpassError>;
+    fn into_response_msg_type(self) -> Result<ResponseMsgType, RosenpassError>;
+}
+
+impl RawMsgTypeExt for RawMsgType {
+    fn into_request_msg_type(self) -> Result<RequestMsgType, RosenpassError> {
+        self.try_into()
+    }
+
+    fn into_response_msg_type(self) -> Result<ResponseMsgType, RosenpassError> {
+        self.try_into()
+    }
+}
+
+pub trait RefMakerRawMsgTypeExt {
+    fn parse_request_msg_type(self) -> anyhow::Result<RequestMsgType>;
+    fn parse_response_msg_type(self) -> anyhow::Result<ResponseMsgType>;
+}
+
+impl<B: ByteSlice> RefMakerRawMsgTypeExt for RefMaker<B, RawMsgType> {
+    fn parse_request_msg_type(self) -> anyhow::Result<RequestMsgType> {
+        Ok(self.parse()?.read().try_into()?)
+    }
+
+    fn parse_response_msg_type(self) -> anyhow::Result<ResponseMsgType> {
+        Ok(self.parse()?.read().try_into()?)
+    }
+}

rosenpass/src/api/boilerplate/mod.rs

@@ -0,0 +1,17 @@
+mod byte_slice_ext;
+mod message_trait;
+mod message_type;
+mod payload;
+mod request_ref;
+mod request_response;
+mod response_ref;
+mod server;
+
+pub use byte_slice_ext::*;
+pub use message_trait::*;
+pub use message_type::*;
+pub use payload::*;
+pub use request_ref::*;
+pub use request_response::*;
+pub use response_ref::*;
+pub use server::*;

rosenpass/src/api/boilerplate/payload.rs

@@ -0,0 +1,351 @@
+use rosenpass_util::zerocopy::ZerocopyMutSliceExt;
+use zerocopy::{AsBytes, ByteSliceMut, FromBytes, FromZeroes, Ref};
+
+use super::{Message, RawMsgType, RequestMsgType, ResponseMsgType};
+
+/// Size required to fit any message in binary form
+pub const MAX_REQUEST_LEN: usize = 2500; // TODO fix this
+pub const MAX_RESPONSE_LEN: usize = 2500; // TODO fix this
+pub const MAX_REQUEST_FDS: usize = 2;
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct Envelope<M: AsBytes + FromBytes> {
+    /// Which message this is
+    pub msg_type: RawMsgType,
+    /// The actual Paylod
+    pub payload: M,
+}
+
+pub type RequestEnvelope<M> = Envelope<M>;
+pub type ResponseEnvelope<M> = Envelope<M>;
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct PingRequestPayload {
+    /// Randomly generated connection id
+    pub echo: [u8; 256],
+}
+
+pub type PingRequest = RequestEnvelope<PingRequestPayload>;
+
+impl PingRequest {
+    pub fn new(echo: [u8; 256]) -> Self {
+        Self::from_payload(PingRequestPayload { echo })
+    }
+}
+
+impl Message for PingRequest {
+    type Payload = PingRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::Ping;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct PingResponsePayload {
+    /// Randomly generated connection id
+    pub echo: [u8; 256],
+}
+
+pub type PingResponse = ResponseEnvelope<PingResponsePayload>;
+
+impl PingResponse {
+    pub fn new(echo: [u8; 256]) -> Self {
+        Self::from_payload(PingResponsePayload { echo })
+    }
+}
+
+impl Message for PingResponse {
+    type Payload = PingResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::Ping;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct SupplyKeypairRequestPayload {}
+
+pub type SupplyKeypairRequest = RequestEnvelope<SupplyKeypairRequestPayload>;
+
+impl Default for SupplyKeypairRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SupplyKeypairRequest {
+    pub fn new() -> Self {
+        Self::from_payload(SupplyKeypairRequestPayload {})
+    }
+}
+
+impl Message for SupplyKeypairRequest {
+    type Payload = SupplyKeypairRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::SupplyKeypair;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod supply_keypair_response_status {
+    pub const OK: u128 = 0;
+    pub const KEYPAIR_ALREADY_SUPPLIED: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+    pub const INVALID_REQUEST: u128 = 3;
+    pub const IO_ERROR: u128 = 4;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct SupplyKeypairResponsePayload {
+    pub status: u128,
+}
+
+pub type SupplyKeypairResponse = ResponseEnvelope<SupplyKeypairResponsePayload>;
+
+impl SupplyKeypairResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(SupplyKeypairResponsePayload { status })
+    }
+}
+
+impl Message for SupplyKeypairResponse {
+    type Payload = SupplyKeypairResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::SupplyKeypair;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddListenSocketRequestPayload {}
+
+pub type AddListenSocketRequest = RequestEnvelope<AddListenSocketRequestPayload>;
+
+impl Default for AddListenSocketRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AddListenSocketRequest {
+    pub fn new() -> Self {
+        Self::from_payload(AddListenSocketRequestPayload {})
+    }
+}
+
+impl Message for AddListenSocketRequest {
+    type Payload = AddListenSocketRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::AddListenSocket;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod add_listen_socket_response_status {
+    pub const OK: u128 = 0;
+    pub const INVALID_REQUEST: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddListenSocketResponsePayload {
+    pub status: u128,
+}
+
+pub type AddListenSocketResponse = ResponseEnvelope<AddListenSocketResponsePayload>;
+
+impl AddListenSocketResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(AddListenSocketResponsePayload { status })
+    }
+}
+
+impl Message for AddListenSocketResponse {
+    type Payload = AddListenSocketResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::AddListenSocket;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddPskBrokerRequestPayload {}
+
+pub type AddPskBrokerRequest = RequestEnvelope<AddPskBrokerRequestPayload>;
+
+impl Default for AddPskBrokerRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AddPskBrokerRequest {
+    pub fn new() -> Self {
+        Self::from_payload(AddPskBrokerRequestPayload {})
+    }
+}
+
+impl Message for AddPskBrokerRequest {
+    type Payload = AddPskBrokerRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::AddPskBroker;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod add_psk_broker_response_status {
+    pub const OK: u128 = 0;
+    pub const INVALID_REQUEST: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddPskBrokerResponsePayload {
+    pub status: u128,
+}
+
+pub type AddPskBrokerResponse = ResponseEnvelope<AddPskBrokerResponsePayload>;
+
+impl AddPskBrokerResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(AddPskBrokerResponsePayload { status })
+    }
+}
+
+impl Message for AddPskBrokerResponse {
+    type Payload = AddPskBrokerResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::AddPskBroker;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}

rosenpass/src/api/boilerplate/request_ref.rs

@@ -0,0 +1,146 @@
+use anyhow::ensure;
+
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{ByteSliceRefExt, MessageAttributes, PingRequest, RequestMsgType};
+
+struct RequestRefMaker<B> {
+    buf: B,
+    msg_type: RequestMsgType,
+}
+
+impl<B: ByteSlice> RequestRef<B> {
+    pub fn parse(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.parse()
+    }
+
+    pub fn parse_from_prefix(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.from_prefix()?.parse()
+    }
+
+    pub fn parse_from_suffix(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.from_suffix()?.parse()
+    }
+
+    pub fn message_type(&self) -> RequestMsgType {
+        match self {
+            Self::Ping(_) => RequestMsgType::Ping,
+            Self::SupplyKeypair(_) => RequestMsgType::SupplyKeypair,
+            Self::AddListenSocket(_) => RequestMsgType::AddListenSocket,
+            Self::AddPskBroker(_) => RequestMsgType::AddPskBroker,
+        }
+    }
+}
+
+impl<B> From<Ref<B, PingRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, PingRequest>) -> Self {
+        Self::Ping(v)
+    }
+}
+
+impl<B> From<Ref<B, super::SupplyKeypairRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::SupplyKeypairRequest>) -> Self {
+        Self::SupplyKeypair(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddListenSocketRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::AddListenSocketRequest>) -> Self {
+        Self::AddListenSocket(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddPskBrokerRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::AddPskBrokerRequest>) -> Self {
+        Self::AddPskBroker(v)
+    }
+}
+
+impl<B: ByteSlice> RequestRefMaker<B> {
+    fn new(buf: B) -> anyhow::Result<Self> {
+        let msg_type = buf.deref().request_msg_type_from_prefix()?;
+        Ok(Self { buf, msg_type })
+    }
+
+    fn target_size(&self) -> usize {
+        self.msg_type.message_size()
+    }
+
+    fn parse(self) -> anyhow::Result<RequestRef<B>> {
+        Ok(match self.msg_type {
+            RequestMsgType::Ping => RequestRef::Ping(self.buf.ping_request()?),
+            RequestMsgType::SupplyKeypair => {
+                RequestRef::SupplyKeypair(self.buf.supply_keypair_request()?)
+            }
+            RequestMsgType::AddListenSocket => {
+                RequestRef::AddListenSocket(self.buf.add_listen_socket_request()?)
+            }
+            RequestMsgType::AddPskBroker => {
+                RequestRef::AddPskBroker(self.buf.add_psk_broker_request()?)
+            }
+        })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_prefix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_suffix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.buf.len() - self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    pub fn ensure_fit(&self) -> anyhow::Result<()> {
+        let have = self.buf.len();
+        let need = self.target_size();
+        ensure!(
+            need <= have,
+            "Buffer is undersized at {have} bytes (need {need} bytes)!"
+        );
+        Ok(())
+    }
+}
+
+pub enum RequestRef<B> {
+    Ping(Ref<B, PingRequest>),
+    SupplyKeypair(Ref<B, super::SupplyKeypairRequest>),
+    AddListenSocket(Ref<B, super::AddListenSocketRequest>),
+    AddPskBroker(Ref<B, super::AddPskBrokerRequest>),
+}
+
+impl<B> RequestRef<B>
+where
+    B: ByteSlice,
+{
+    pub fn bytes(&self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes(),
+            Self::SupplyKeypair(r) => r.bytes(),
+            Self::AddListenSocket(r) => r.bytes(),
+            Self::AddPskBroker(r) => r.bytes(),
+        }
+    }
+}
+
+impl<B> RequestRef<B>
+where
+    B: ByteSliceMut,
+{
+    pub fn bytes_mut(&mut self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes_mut(),
+            Self::SupplyKeypair(r) => r.bytes_mut(),
+            Self::AddListenSocket(r) => r.bytes_mut(),
+            Self::AddPskBroker(r) => r.bytes_mut(),
+        }
+    }
+}

rosenpass/src/api/boilerplate/request_response.rs

@@ -0,0 +1,190 @@
+use rosenpass_util::zerocopy::{
+    RefMaker, ZerocopyEmancipateExt, ZerocopyEmancipateMutExt, ZerocopySliceExt,
+};
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{Message, PingRequest, PingResponse};
+use super::{RequestRef, ResponseRef, ZerocopyResponseMakerSetupMessageExt};
+
+pub trait RequestMsg: Sized + Message {
+    type ResponseMsg: ResponseMsg;
+
+    fn zk_response_maker<B: ByteSlice>(buf: B) -> RefMaker<B, Self::ResponseMsg> {
+        buf.zk_ref_maker()
+    }
+
+    fn setup_response<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).setup_msg()
+    }
+
+    fn setup_response_from_prefix<B: ByteSliceMut>(
+        buf: B,
+    ) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).from_prefix()?.setup_msg()
+    }
+
+    fn setup_response_from_suffix<B: ByteSliceMut>(
+        buf: B,
+    ) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).from_prefix()?.setup_msg()
+    }
+}
+
+pub trait ResponseMsg: Message {
+    type RequestMsg: RequestMsg;
+}
+
+impl RequestMsg for PingRequest {
+    type ResponseMsg = PingResponse;
+}
+
+impl ResponseMsg for PingResponse {
+    type RequestMsg = PingRequest;
+}
+
+impl RequestMsg for super::SupplyKeypairRequest {
+    type ResponseMsg = super::SupplyKeypairResponse;
+}
+
+impl ResponseMsg for super::SupplyKeypairResponse {
+    type RequestMsg = super::SupplyKeypairRequest;
+}
+
+impl RequestMsg for super::AddListenSocketRequest {
+    type ResponseMsg = super::AddListenSocketResponse;
+}
+
+impl ResponseMsg for super::AddListenSocketResponse {
+    type RequestMsg = super::AddListenSocketRequest;
+}
+
+impl RequestMsg for super::AddPskBrokerRequest {
+    type ResponseMsg = super::AddPskBrokerResponse;
+}
+
+impl ResponseMsg for super::AddPskBrokerResponse {
+    type RequestMsg = super::AddPskBrokerRequest;
+}
+
+pub type PingPair<B1, B2> = (Ref<B1, PingRequest>, Ref<B2, PingResponse>);
+pub type SupplyKeypairPair<B1, B2> = (
+    Ref<B1, super::SupplyKeypairRequest>,
+    Ref<B2, super::SupplyKeypairResponse>,
+);
+pub type AddListenSocketPair<B1, B2> = (
+    Ref<B1, super::AddListenSocketRequest>,
+    Ref<B2, super::AddListenSocketResponse>,
+);
+pub type AddPskBrokerPair<B1, B2> = (
+    Ref<B1, super::AddPskBrokerRequest>,
+    Ref<B2, super::AddPskBrokerResponse>,
+);
+
+pub enum RequestResponsePair<B1, B2> {
+    Ping(PingPair<B1, B2>),
+    SupplyKeypair(SupplyKeypairPair<B1, B2>),
+    AddListenSocket(AddListenSocketPair<B1, B2>),
+    AddPskBroker(AddPskBrokerPair<B1, B2>),
+}
+
+impl<B1, B2> From<PingPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: PingPair<B1, B2>) -> Self {
+        RequestResponsePair::Ping(v)
+    }
+}
+
+impl<B1, B2> From<SupplyKeypairPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: SupplyKeypairPair<B1, B2>) -> Self {
+        RequestResponsePair::SupplyKeypair(v)
+    }
+}
+
+impl<B1, B2> From<AddListenSocketPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: AddListenSocketPair<B1, B2>) -> Self {
+        RequestResponsePair::AddListenSocket(v)
+    }
+}
+
+impl<B1, B2> From<AddPskBrokerPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: AddPskBrokerPair<B1, B2>) -> Self {
+        RequestResponsePair::AddPskBroker(v)
+    }
+}
+
+impl<B1, B2> RequestResponsePair<B1, B2>
+where
+    B1: ByteSlice,
+    B2: ByteSlice,
+{
+    pub fn both(&self) -> (RequestRef<&[u8]>, ResponseRef<&[u8]>) {
+        match self {
+            Self::Ping((req, res)) => {
+                let req = RequestRef::Ping(req.emancipate());
+                let res = ResponseRef::Ping(res.emancipate());
+                (req, res)
+            }
+            Self::SupplyKeypair((req, res)) => {
+                let req = RequestRef::SupplyKeypair(req.emancipate());
+                let res = ResponseRef::SupplyKeypair(res.emancipate());
+                (req, res)
+            }
+            Self::AddListenSocket((req, res)) => {
+                let req = RequestRef::AddListenSocket(req.emancipate());
+                let res = ResponseRef::AddListenSocket(res.emancipate());
+                (req, res)
+            }
+            Self::AddPskBroker((req, res)) => {
+                let req = RequestRef::AddPskBroker(req.emancipate());
+                let res = ResponseRef::AddPskBroker(res.emancipate());
+                (req, res)
+            }
+        }
+    }
+
+    pub fn request(&self) -> RequestRef<&[u8]> {
+        self.both().0
+    }
+
+    pub fn response(&self) -> ResponseRef<&[u8]> {
+        self.both().1
+    }
+}
+
+impl<B1, B2> RequestResponsePair<B1, B2>
+where
+    B1: ByteSliceMut,
+    B2: ByteSliceMut,
+{
+    pub fn both_mut(&mut self) -> (RequestRef<&mut [u8]>, ResponseRef<&mut [u8]>) {
+        match self {
+            Self::Ping((req, res)) => {
+                let req = RequestRef::Ping(req.emancipate_mut());
+                let res = ResponseRef::Ping(res.emancipate_mut());
+                (req, res)
+            }
+            Self::SupplyKeypair((req, res)) => {
+                let req = RequestRef::SupplyKeypair(req.emancipate_mut());
+                let res = ResponseRef::SupplyKeypair(res.emancipate_mut());
+                (req, res)
+            }
+            Self::AddListenSocket((req, res)) => {
+                let req = RequestRef::AddListenSocket(req.emancipate_mut());
+                let res = ResponseRef::AddListenSocket(res.emancipate_mut());
+                (req, res)
+            }
+            Self::AddPskBroker((req, res)) => {
+                let req = RequestRef::AddPskBroker(req.emancipate_mut());
+                let res = ResponseRef::AddPskBroker(res.emancipate_mut());
+                (req, res)
+            }
+        }
+    }
+
+    pub fn request_mut(&mut self) -> RequestRef<&mut [u8]> {
+        self.both_mut().0
+    }
+
+    pub fn response_mut(&mut self) -> ResponseRef<&mut [u8]> {
+        self.both_mut().1
+    }
+}

rosenpass/src/api/boilerplate/response_ref.rs

@@ -0,0 +1,147 @@
+// TODO: This is copied verbatim from ResponseRef…not pretty
+use anyhow::ensure;
+
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{ByteSliceRefExt, MessageAttributes, PingResponse, ResponseMsgType};
+
+struct ResponseRefMaker<B> {
+    buf: B,
+    msg_type: ResponseMsgType,
+}
+
+impl<B: ByteSlice> ResponseRef<B> {
+    pub fn parse(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.parse()
+    }
+
+    pub fn parse_from_prefix(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.from_prefix()?.parse()
+    }
+
+    pub fn parse_from_suffix(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.from_suffix()?.parse()
+    }
+
+    pub fn message_type(&self) -> ResponseMsgType {
+        match self {
+            Self::Ping(_) => ResponseMsgType::Ping,
+            Self::SupplyKeypair(_) => ResponseMsgType::SupplyKeypair,
+            Self::AddListenSocket(_) => ResponseMsgType::AddListenSocket,
+            Self::AddPskBroker(_) => ResponseMsgType::AddPskBroker,
+        }
+    }
+}
+
+impl<B> From<Ref<B, PingResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, PingResponse>) -> Self {
+        Self::Ping(v)
+    }
+}
+
+impl<B> From<Ref<B, super::SupplyKeypairResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::SupplyKeypairResponse>) -> Self {
+        Self::SupplyKeypair(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddListenSocketResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::AddListenSocketResponse>) -> Self {
+        Self::AddListenSocket(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddPskBrokerResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::AddPskBrokerResponse>) -> Self {
+        Self::AddPskBroker(v)
+    }
+}
+
+impl<B: ByteSlice> ResponseRefMaker<B> {
+    fn new(buf: B) -> anyhow::Result<Self> {
+        let msg_type = buf.deref().response_msg_type_from_prefix()?;
+        Ok(Self { buf, msg_type })
+    }
+
+    fn target_size(&self) -> usize {
+        self.msg_type.message_size()
+    }
+
+    fn parse(self) -> anyhow::Result<ResponseRef<B>> {
+        Ok(match self.msg_type {
+            ResponseMsgType::Ping => ResponseRef::Ping(self.buf.ping_response()?),
+            ResponseMsgType::SupplyKeypair => {
+                ResponseRef::SupplyKeypair(self.buf.supply_keypair_response()?)
+            }
+            ResponseMsgType::AddListenSocket => {
+                ResponseRef::AddListenSocket(self.buf.add_listen_socket_response()?)
+            }
+            ResponseMsgType::AddPskBroker => {
+                ResponseRef::AddPskBroker(self.buf.add_psk_broker_response()?)
+            }
+        })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_prefix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_suffix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.buf.len() - self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    pub fn ensure_fit(&self) -> anyhow::Result<()> {
+        let have = self.buf.len();
+        let need = self.target_size();
+        ensure!(
+            need <= have,
+            "Buffer is undersized at {have} bytes (need {need} bytes)!"
+        );
+        Ok(())
+    }
+}
+
+pub enum ResponseRef<B> {
+    Ping(Ref<B, PingResponse>),
+    SupplyKeypair(Ref<B, super::SupplyKeypairResponse>),
+    AddListenSocket(Ref<B, super::AddListenSocketResponse>),
+    AddPskBroker(Ref<B, super::AddPskBrokerResponse>),
+}
+
+impl<B> ResponseRef<B>
+where
+    B: ByteSlice,
+{
+    pub fn bytes(&self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes(),
+            Self::SupplyKeypair(r) => r.bytes(),
+            Self::AddListenSocket(r) => r.bytes(),
+            Self::AddPskBroker(r) => r.bytes(),
+        }
+    }
+}
+
+impl<B> ResponseRef<B>
+where
+    B: ByteSliceMut,
+{
+    pub fn bytes_mut(&mut self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes_mut(),
+            Self::SupplyKeypair(r) => r.bytes_mut(),
+            Self::AddListenSocket(r) => r.bytes_mut(),
+            Self::AddPskBroker(r) => r.bytes_mut(),
+        }
+    }
+}

rosenpass/src/api/boilerplate/server.rs

@@ -0,0 +1,94 @@
+use super::{ByteSliceRefExt, Message, PingRequest, PingResponse, RequestRef, RequestResponsePair};
+use std::{collections::VecDeque, os::fd::OwnedFd};
+use zerocopy::{ByteSlice, ByteSliceMut};
+
+pub trait Server {
+    fn ping(
+        &mut self,
+        req: &PingRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut PingResponse,
+    ) -> anyhow::Result<()>;
+
+    fn supply_keypair(
+        &mut self,
+        req: &super::SupplyKeypairRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::SupplyKeypairResponse,
+    ) -> anyhow::Result<()>;
+
+    fn add_listen_socket(
+        &mut self,
+        req: &super::AddListenSocketRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::AddListenSocketResponse,
+    ) -> anyhow::Result<()>;
+
+    fn add_psk_broker(
+        &mut self,
+        req: &super::AddPskBrokerRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::AddPskBrokerResponse,
+    ) -> anyhow::Result<()>;
+
+    fn dispatch<ReqBuf, ResBuf>(
+        &mut self,
+        p: &mut RequestResponsePair<ReqBuf, ResBuf>,
+        req_fds: &mut VecDeque<OwnedFd>,
+    ) -> anyhow::Result<()>
+    where
+        ReqBuf: ByteSlice,
+        ResBuf: ByteSliceMut,
+    {
+        match p {
+            RequestResponsePair::Ping((req, res)) => self.ping(req, req_fds, res),
+            RequestResponsePair::SupplyKeypair((req, res)) => {
+                self.supply_keypair(req, req_fds, res)
+            }
+            RequestResponsePair::AddListenSocket((req, res)) => {
+                self.add_listen_socket(req, req_fds, res)
+            }
+            RequestResponsePair::AddPskBroker((req, res)) => self.add_psk_broker(req, req_fds, res),
+        }
+    }
+
+    fn handle_message<ReqBuf, ResBuf>(
+        &mut self,
+        req: ReqBuf,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: ResBuf,
+    ) -> anyhow::Result<usize>
+    where
+        ReqBuf: ByteSlice,
+        ResBuf: ByteSliceMut,
+    {
+        let req = req.parse_request_from_prefix()?;
+        // TODO: This is not pretty; This match should be moved into RequestRef
+        let mut pair = match req {
+            RequestRef::Ping(req) => {
+                let mut res = res.ping_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::Ping((req, res))
+            }
+            RequestRef::SupplyKeypair(req) => {
+                let mut res = res.supply_keypair_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::SupplyKeypair((req, res))
+            }
+            RequestRef::AddListenSocket(req) => {
+                let mut res = res.add_listen_socket_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::AddListenSocket((req, res))
+            }
+            RequestRef::AddPskBroker(req) => {
+                let mut res = res.add_psk_broker_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::AddPskBroker((req, res))
+            }
+        };
+        self.dispatch(&mut pair, req_fds)?;
+
+        let res_len = pair.response().bytes().len();
+        Ok(res_len)
+    }
+}

rosenpass/src/api/cli.rs

@@ -0,0 +1,40 @@
+use std::path::PathBuf;
+
+use clap::Args;
+
+use crate::config::Rosenpass as RosenpassConfig;
+
+use super::config::ApiConfig;
+
+#[cfg(feature = "experiment_api")]
+#[derive(Args, Debug)]
+pub struct ApiCli {
+    /// Where in the file-system to create the unix socket the rosenpass API will be listening for
+    /// connections on.
+    #[arg(long)]
+    api_listen_path: Vec<PathBuf>,
+
+    /// When rosenpass is called from another process, the other process can open and bind the
+    /// unix socket for the Rosenpass API to use themselves, passing it to this process. In Rust this can be achieved
+    /// using the [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate.
+    #[arg(long)]
+    api_listen_fd: Vec<i32>,
+
+    /// When rosenpass is called from another process, the other process can connect the unix socket for the API
+    /// themselves, for instance using the `socketpair(2)` system call.
+    #[arg(long)]
+    api_stream_fd: Vec<i32>,
+}
+
+impl ApiCli {
+    pub fn apply_to_config(&self, cfg: &mut RosenpassConfig) -> anyhow::Result<()> {
+        self.apply_to_api_config(&mut cfg.api)
+    }
+
+    pub fn apply_to_api_config(&self, cfg: &mut ApiConfig) -> anyhow::Result<()> {
+        cfg.listen_path.extend_from_slice(&self.api_listen_path);
+        cfg.listen_fd.extend_from_slice(&self.api_listen_fd);
+        cfg.stream_fd.extend_from_slice(&self.api_stream_fd);
+        Ok(())
+    }
+}

rosenpass/src/api/config.rs

@@ -0,0 +1,49 @@
+use std::path::PathBuf;
+
+use mio::net::UnixListener;
+use rosenpass_util::mio::{UnixListenerExt, UnixStreamExt};
+use serde::{Deserialize, Serialize};
+
+use crate::app_server::AppServer;
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct ApiConfig {
+    /// Where in the file-system to create the unix socket the rosenpass API will be listening for
+    /// connections on
+    pub listen_path: Vec<PathBuf>,
+
+    /// When rosenpass is called from another process, the other process can open and bind the
+    /// unix socket for the Rosenpass API to use themselves, passing it to this process. In Rust this can be achieved
+    /// using the [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate.
+    pub listen_fd: Vec<i32>,
+
+    /// When rosenpass is called from another process, the other process can connect the unix socket for the API
+    /// themselves, for instance using the `socketpair(2)` system call.
+    pub stream_fd: Vec<i32>,
+}
+
+impl ApiConfig {
+    pub fn apply_to_app_server(&self, srv: &mut AppServer) -> anyhow::Result<()> {
+        for path in self.listen_path.iter() {
+            srv.add_api_listener(UnixListener::bind(path)?)?;
+        }
+
+        for fd in self.listen_fd.iter() {
+            srv.add_api_listener(UnixListenerExt::claim_fd(*fd)?)?;
+        }
+
+        for fd in self.stream_fd.iter() {
+            srv.add_api_connection(UnixStreamExt::claim_fd(*fd)?)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn count_api_sources(&self) -> usize {
+        self.listen_path.len() + self.listen_fd.len() + self.stream_fd.len()
+    }
+
+    pub fn has_api_sources(&self) -> bool {
+        self.count_api_sources() > 0
+    }
+}

rosenpass/src/api/mio/connection.rs

@@ -0,0 +1,321 @@
+use std::borrow::{Borrow, BorrowMut};
+use std::collections::VecDeque;
+use std::os::fd::OwnedFd;
+
+use mio::net::UnixStream;
+use rosenpass_secret_memory::Secret;
+use rosenpass_util::mio::ReadWithFileDescriptors;
+use rosenpass_util::{
+    io::{IoResultKindHintExt, TryIoResultKindHintExt},
+    length_prefix_encoding::{
+        decoder::{self as lpe_decoder, LengthPrefixDecoder},
+        encoder::{self as lpe_encoder, LengthPrefixEncoder},
+    },
+    mio::interest::RW as MIO_RW,
+};
+use zeroize::Zeroize;
+
+use crate::api::MAX_REQUEST_FDS;
+use crate::{api::Server, app_server::AppServer};
+
+use super::super::{ApiHandler, ApiHandlerContext};
+
+#[derive(Debug)]
+struct SecretBuffer<const N: usize>(pub Secret<N>);
+
+impl<const N: usize> SecretBuffer<N> {
+    fn new() -> Self {
+        Self(Secret::zero())
+    }
+}
+
+impl<const N: usize> Borrow<[u8]> for SecretBuffer<N> {
+    fn borrow(&self) -> &[u8] {
+        self.0.secret()
+    }
+}
+
+impl<const N: usize> BorrowMut<[u8]> for SecretBuffer<N> {
+    fn borrow_mut(&mut self) -> &mut [u8] {
+        self.0.secret_mut()
+    }
+}
+// TODO: Unfortunately, zerocopy is quite particular about alignment, hence the 4096
+type ReadBuffer = LengthPrefixDecoder<SecretBuffer<4096>>;
+type WriteBuffer = LengthPrefixEncoder<SecretBuffer<4096>>;
+type ReadFdBuffer = VecDeque<OwnedFd>;
+
+#[derive(Debug)]
+struct MioConnectionBuffers {
+    read_buffer: ReadBuffer,
+    write_buffer: WriteBuffer,
+    read_fd_buffer: ReadFdBuffer,
+}
+
+#[derive(Debug)]
+pub struct MioConnection {
+    io: UnixStream,
+    mio_token: mio::Token,
+    invalid_read: bool,
+    buffers: Option<MioConnectionBuffers>,
+    api_handler: ApiHandler,
+}
+
+impl MioConnection {
+    pub fn new(app_server: &mut AppServer, mut io: UnixStream) -> std::io::Result<Self> {
+        let mio_token = app_server.mio_token_dispenser.dispense();
+        app_server
+            .mio_poll
+            .registry()
+            .register(&mut io, mio_token, MIO_RW)?;
+
+        let invalid_read = false;
+        let read_buffer = LengthPrefixDecoder::new(SecretBuffer::new());
+        let write_buffer = LengthPrefixEncoder::from_buffer(SecretBuffer::new());
+        let read_fd_buffer = VecDeque::new();
+        let buffers = Some(MioConnectionBuffers {
+            read_buffer,
+            write_buffer,
+            read_fd_buffer,
+        });
+        let api_state = ApiHandler::new();
+        Ok(Self {
+            io,
+            mio_token,
+            invalid_read,
+            buffers,
+            api_handler: api_state,
+        })
+    }
+
+    pub fn shoud_close(&self) -> bool {
+        let exhausted = self
+            .buffers
+            .as_ref()
+            .map(|b| b.write_buffer.exhausted())
+            .unwrap_or(false);
+        self.invalid_read && exhausted
+    }
+
+    pub fn close(mut self, app_server: &mut AppServer) -> anyhow::Result<()> {
+        app_server.mio_poll.registry().deregister(&mut self.io)?;
+        Ok(())
+    }
+
+    pub fn mio_token(&self) -> mio::Token {
+        self.mio_token
+    }
+}
+
+pub trait MioConnectionContext {
+    fn mio_connection(&self) -> &MioConnection;
+    fn app_server(&self) -> &AppServer;
+    fn mio_connection_mut(&mut self) -> &mut MioConnection;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+
+    fn poll(&mut self) -> anyhow::Result<()> {
+        macro_rules! short {
+            ($e:expr) => {
+                match $e {
+                    None => return Ok(()),
+                    Some(()) => {}
+                }
+            };
+        }
+
+        // All of these functions return an error, None ("operation incomplete")
+        // or some ("operation complete, keep processing")
+        short!(self.flush_write_buffer()?); // Flush last message
+        short!(self.recv()?); // Receive new message
+        short!(self.handle_incoming_message()?); // Process new message with API
+        short!(self.flush_write_buffer()?); // Begin flushing response
+
+        Ok(())
+    }
+
+    fn handle_incoming_message(&mut self) -> anyhow::Result<Option<()>> {
+        self.with_buffers_stolen(|this, bufs| {
+            // Acquire request & response. Caller is responsible to make sure
+            // that read buffer holds a message and that write buffer is cleared.
+            // Hence the unwraps and assertions
+            assert!(bufs.write_buffer.exhausted());
+            let req = bufs.read_buffer.message().unwrap().unwrap();
+            let req_fds = &mut bufs.read_fd_buffer;
+            let res = bufs.write_buffer.buffer_bytes_mut();
+
+            // Call API handler
+            // Transitive trait implementations: MioConnectionContext -> ApiHandlerContext -> as ApiServer
+            let response_len = this.handle_message(req, req_fds, res)?;
+
+            bufs.write_buffer
+                .restart_write_with_new_message(response_len)?;
+            bufs.read_buffer.zeroize(); // clear for new message to read
+            bufs.read_fd_buffer.clear();
+
+            Ok(Some(()))
+        })
+    }
+
+    fn flush_write_buffer(&mut self) -> anyhow::Result<Option<()>> {
+        if self.write_buf_mut().exhausted() {
+            return Ok(Some(()));
+        }
+
+        use lpe_encoder::WriteToIoReturn as Ret;
+        use std::io::ErrorKind as K;
+
+        loop {
+            let conn = self.mio_connection_mut();
+            let bufs = conn.buffers.as_mut().unwrap();
+
+            let sock = &conn.io;
+            let write_buf = &mut bufs.write_buffer;
+
+            match write_buf.write_to_stdio(sock).io_err_kind_hint() {
+                // Done
+                Ok(Ret { done: true, .. }) => {
+                    write_buf.zeroize(); // clear for new message to write
+                    break Ok(Some(()));
+                }
+
+                // Would block
+                Ok(Ret {
+                    bytes_written: 0, ..
+                }) => break Ok(None),
+                Err((_e, K::WouldBlock)) => break Ok(None),
+
+                // Just continue
+                Ok(_) => continue, /* Ret { bytes_written > 0, done = false } acc. to previous cases*/
+                Err((_e, K::Interrupted)) => continue,
+
+                // Other errors
+                Err((e, _ek)) => Err(e)?,
+            }
+        }
+    }
+
+    fn recv(&mut self) -> anyhow::Result<Option<()>> {
+        if !self.write_buf_mut().exhausted() || self.mio_connection().invalid_read {
+            return Ok(None);
+        }
+
+        use lpe_decoder::{ReadFromIoError as E, ReadFromIoReturn as Ret};
+        use std::io::ErrorKind as K;
+
+        loop {
+            let conn = self.mio_connection_mut();
+            let bufs = conn.buffers.as_mut().unwrap();
+
+            let read_buf = &mut bufs.read_buffer;
+            let read_fd_buf = &mut bufs.read_fd_buffer;
+
+            let sock = &conn.io;
+            let fd_passing_sock = ReadWithFileDescriptors::<MAX_REQUEST_FDS, UnixStream, _, _>::new(
+                sock,
+                read_fd_buf,
+            );
+
+            match read_buf
+                .read_from_stdio(fd_passing_sock)
+                .try_io_err_kind_hint()
+            {
+                // We actually received a proper message
+                // (Impl below match to appease borrow checker)
+                Ok(Ret {
+                    message: Some(_msg),
+                    ..
+                }) => break Ok(Some(())),
+
+                // Message does not fit in buffer
+                Err((e @ E::MessageTooLargeError { .. }, _)) => {
+                    log::warn!("Received message on API that was too big to fit in our buffers; \
+                            looks like the client is broken. Stopping to process messages of the client.\n\
+                            Error: {e:?}");
+                    conn.invalid_read = true; // Closed mio_manager
+                    break Ok(None);
+                }
+
+                // Would block
+                Ok(Ret { bytes_read: 0, .. }) => break Ok(None),
+                Err((_, Some(K::WouldBlock))) => break Ok(None),
+
+                // Just keep going
+                Ok(Ret { bytes_read: _, .. }) => continue,
+                Err((_, Some(K::Interrupted))) => continue,
+
+                // Other IO Error (just pass on to the caller)
+                Err((E::IoError(e), _)) => {
+                    log::warn!(
+                        "IO error while trying to read message from API socket. \
+                            The connection is broken. Stopping to process messages of the client.\n\
+                            Error: {e:?}"
+                    );
+                    conn.invalid_read = true; // closed later by mio_manager
+                    break Err(e.into());
+                }
+            };
+        }
+    }
+
+    fn mio_token(&self) -> mio::Token {
+        self.mio_connection().mio_token()
+    }
+
+    fn should_close(&self) -> bool {
+        self.mio_connection().shoud_close()
+    }
+}
+
+trait MioConnectionContextPrivate: MioConnectionContext {
+    fn steal_buffers(&mut self) -> MioConnectionBuffers {
+        self.mio_connection_mut().buffers.take().unwrap()
+    }
+
+    fn return_buffers(&mut self, buffers: MioConnectionBuffers) {
+        let opt = &mut self.mio_connection_mut().buffers;
+        assert!(opt.is_none());
+        let _ = opt.insert(buffers);
+    }
+
+    fn with_buffers_stolen<R, F: FnOnce(&mut Self, &mut MioConnectionBuffers) -> R>(
+        &mut self,
+        f: F,
+    ) -> R {
+        let mut bufs = self.steal_buffers();
+        let res = f(self, &mut bufs);
+        self.return_buffers(bufs);
+        res
+    }
+
+    fn write_buf_mut(&mut self) -> &mut WriteBuffer {
+        self.mio_connection_mut()
+            .buffers
+            .as_mut()
+            .unwrap()
+            .write_buffer
+            .borrow_mut()
+    }
+}
+
+impl<T> MioConnectionContextPrivate for T where T: ?Sized + MioConnectionContext {}
+
+impl<T> ApiHandlerContext for T
+where
+    T: ?Sized + MioConnectionContext,
+{
+    fn api_handler(&self) -> &ApiHandler {
+        &self.mio_connection().api_handler
+    }
+
+    fn app_server(&self) -> &AppServer {
+        MioConnectionContext::app_server(self)
+    }
+
+    fn api_handler_mut(&mut self) -> &mut ApiHandler {
+        &mut self.mio_connection_mut().api_handler
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        MioConnectionContext::app_server_mut(self)
+    }
+}

rosenpass/src/api/mio/manager.rs

@@ -0,0 +1,173 @@
+use std::{borrow::BorrowMut, io};
+
+use mio::net::{UnixListener, UnixStream};
+
+use rosenpass_util::{
+    functional::ApplyExt, io::nonblocking_handle_io_errors, mio::interest::RW as MIO_RW,
+};
+
+use crate::app_server::{AppServer, AppServerIoSource};
+
+use super::{MioConnection, MioConnectionContext};
+
+#[derive(Default, Debug)]
+pub struct MioManager {
+    listeners: Vec<UnixListener>,
+    connections: Vec<Option<MioConnection>>,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum MioManagerIoSource {
+    Listener(usize),
+    Connection(usize),
+}
+
+impl MioManager {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+struct MioConnectionFocus<'a, T: ?Sized + MioManagerContext> {
+    ctx: &'a mut T,
+    conn_idx: usize,
+}
+
+impl<'a, T: ?Sized + MioManagerContext> MioConnectionFocus<'a, T> {
+    fn new(ctx: &'a mut T, conn_idx: usize) -> Self {
+        Self { ctx, conn_idx }
+    }
+}
+
+pub trait MioManagerContext {
+    fn mio_manager(&self) -> &MioManager;
+    fn mio_manager_mut(&mut self) -> &mut MioManager;
+    fn app_server(&self) -> &AppServer;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+
+    fn add_listener(&mut self, mut listener: UnixListener) -> io::Result<()> {
+        let srv = self.app_server_mut();
+        let mio_token = srv.mio_token_dispenser.dispense();
+        srv.mio_poll
+            .registry()
+            .register(&mut listener, mio_token, MIO_RW)?;
+        let io_source = self
+            .mio_manager()
+            .listeners
+            .len()
+            .apply(MioManagerIoSource::Listener)
+            .apply(AppServerIoSource::MioManager);
+        self.mio_manager_mut().listeners.push(listener);
+        self.app_server_mut()
+            .register_io_source(mio_token, io_source);
+
+        Ok(())
+    }
+
+    fn add_connection(&mut self, connection: UnixStream) -> io::Result<()> {
+        let connection = MioConnection::new(self.app_server_mut(), connection)?;
+        let mio_token = connection.mio_token();
+        let conns: &mut Vec<Option<MioConnection>> =
+            self.mio_manager_mut().connections.borrow_mut();
+        let idx = conns
+            .iter_mut()
+            .enumerate()
+            .find(|(_, slot)| slot.is_some())
+            .map(|(idx, _)| idx)
+            .unwrap_or(conns.len());
+        conns.insert(idx, Some(connection));
+        let io_source = idx
+            .apply(MioManagerIoSource::Listener)
+            .apply(AppServerIoSource::MioManager);
+        self.app_server_mut()
+            .register_io_source(mio_token, io_source);
+        Ok(())
+    }
+
+    fn poll_particular(&mut self, io_source: MioManagerIoSource) -> anyhow::Result<()> {
+        use MioManagerIoSource as S;
+        match io_source {
+            S::Listener(idx) => self.accept_from(idx)?,
+            S::Connection(idx) => self.poll_particular_connection(idx)?,
+        };
+        Ok(())
+    }
+
+    fn poll(&mut self) -> anyhow::Result<()> {
+        self.accept_connections()?;
+        self.poll_connections()?;
+        Ok(())
+    }
+
+    fn accept_connections(&mut self) -> io::Result<()> {
+        for idx in 0..self.mio_manager_mut().listeners.len() {
+            self.accept_from(idx)?;
+        }
+        Ok(())
+    }
+
+    fn accept_from(&mut self, idx: usize) -> io::Result<()> {
+        // Accept connection until the socket would block or returns another error
+        // TODO: This currently only adds connections--we eventually need the ability to remove
+        // them as well, see the note in connection.rs
+        loop {
+            match nonblocking_handle_io_errors(|| self.mio_manager().listeners[idx].accept())? {
+                None => break,
+                Some((conn, _addr)) => {
+                    self.add_connection(conn)?;
+                }
+            };
+        }
+
+        Ok(())
+    }
+
+    fn poll_connections(&mut self) -> anyhow::Result<()> {
+        for idx in 0..self.mio_manager().connections.len() {
+            self.poll_particular_connection(idx)?;
+        }
+        Ok(())
+    }
+
+    fn poll_particular_connection(&mut self, idx: usize) -> anyhow::Result<()> {
+        if self.mio_manager().connections[idx].is_none() {
+            return Ok(());
+        }
+
+        let mut conn = MioConnectionFocus::new(self, idx);
+        conn.poll()?;
+
+        if conn.should_close() {
+            let conn = self.mio_manager_mut().connections[idx].take().unwrap();
+            let mio_token = conn.mio_token();
+            if let Err(e) = conn.close(self.app_server_mut()) {
+                log::warn!("Error while closing API connection {e:?}");
+            };
+            self.app_server_mut().unregister_io_source(mio_token);
+        }
+
+        Ok(())
+    }
+}
+
+impl<T: ?Sized + MioManagerContext> MioConnectionContext for MioConnectionFocus<'_, T> {
+    fn mio_connection(&self) -> &MioConnection {
+        self.ctx.mio_manager().connections[self.conn_idx]
+            .as_ref()
+            .unwrap()
+    }
+
+    fn app_server(&self) -> &AppServer {
+        self.ctx.app_server()
+    }
+
+    fn mio_connection_mut(&mut self) -> &mut MioConnection {
+        self.ctx.mio_manager_mut().connections[self.conn_idx]
+            .as_mut()
+            .unwrap()
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        self.ctx.app_server_mut()
+    }
+}

rosenpass/src/api/mio/mod.rs

@@ -0,0 +1,5 @@
+mod connection;
+mod manager;
+
+pub use connection::*;
+pub use manager::*;

rosenpass/src/api/mod.rs

@@ -0,0 +1,9 @@
+mod api_handler;
+mod boilerplate;
+
+pub use api_handler::*;
+pub use boilerplate::*;
+
+pub mod cli;
+pub mod config;
+pub mod mio;

rosenpass/src/app_server.rs

@@ -1,5 +1,6 @@
 use anyhow::bail;
 
+use anyhow::Context;
 use anyhow::Result;
 use derive_builder::Builder;
 use log::{error, info, warn};
@@ -7,7 +8,14 @@ use mio::Interest;
 use mio::Token;
 use rosenpass_secret_memory::Public;
 use rosenpass_secret_memory::Secret;
+use rosenpass_util::build::ConstructionSite;
 use rosenpass_util::file::StoreValueB64;
+use rosenpass_util::functional::run;
+use rosenpass_util::functional::ApplyExt;
+use rosenpass_util::io::IoResultKindHintExt;
+use rosenpass_util::io::SubstituteForIoErrorKindExt;
+use rosenpass_util::option::SomeExt;
+use rosenpass_util::result::OkExt;
 use rosenpass_wireguard_broker::WireguardBrokerMio;
 use rosenpass_wireguard_broker::{WireguardBrokerCfg, WG_KEY_LEN};
 use zerocopy::AsBytes;
@@ -15,8 +23,12 @@ use zerocopy::AsBytes;
 use std::cell::Cell;
 
 use std::collections::HashMap;
+use std::collections::VecDeque;
 use std::fmt::Debug;
+use std::io;
+use std::io::stdout;
 use std::io::ErrorKind;
+use std::io::Write;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
@@ -28,6 +40,7 @@ use std::slice;
 use std::time::Duration;
 use std::time::Instant;
 
+use crate::protocol::BuildCryptoServer;
 use crate::protocol::HostIdentification;
 use crate::{
     config::Verbosity,
@@ -63,7 +76,7 @@ pub struct MioTokenDispenser {
 }
 
 impl MioTokenDispenser {
-    fn dispense(&mut self) -> Token {
+    pub fn dispense(&mut self) -> Token {
         let r = self.counter;
         self.counter += 1;
         Token(r)
@@ -72,7 +85,7 @@ impl MioTokenDispenser {
 
 #[derive(Debug, Default)]
 pub struct BrokerStore {
-    store: HashMap<
+    pub store: HashMap<
         Public<BROKER_ID_BYTES>,
         Box<dyn WireguardBrokerMio<Error = anyhow::Error, MioError = anyhow::Error>>,
     >,
@@ -138,15 +151,29 @@ pub struct AppServerTest {
     pub termination_handler: Option<std::sync::mpsc::Receiver<()>>,
 }
 
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum AppServerIoSource {
+    Socket(usize),
+    #[cfg(feature = "experiment_api")]
+    PskBroker(Public<BROKER_ID_BYTES>),
+    #[cfg(feature = "experiment_api")]
+    MioManager(crate::api::mio::MioManagerIoSource),
+}
+
+const EVENT_CAPACITY: usize = 20;
+
 /// Holds the state of the application, namely the external IO
 ///
 /// Responsible for file IO, network IO
 // TODO add user control via unix domain socket and stdin/stdout
 #[derive(Debug)]
 pub struct AppServer {
-    pub crypt: CryptoServer,
+    pub crypto_site: ConstructionSite<BuildCryptoServer, CryptoServer>,
     pub sockets: Vec<mio::net::UdpSocket>,
     pub events: mio::Events,
+    pub short_poll_queue: VecDeque<mio::event::Event>,
+    pub performed_long_poll: bool,
+    pub io_source_index: HashMap<mio::Token, AppServerIoSource>,
     pub mio_poll: mio::Poll,
     pub mio_token_dispenser: MioTokenDispenser,
     pub brokers: BrokerStore,
@@ -159,6 +186,8 @@ pub struct AppServer {
     pub unpolled_count: usize,
     pub last_update_time: Instant,
     pub test_helpers: Option<AppServerTest>,
+    #[cfg(feature = "experiment_api")]
+    pub api_manager: crate::api::mio::MioManager,
 }
 
 /// A socket pointer is an index assigned to a socket;
@@ -507,15 +536,14 @@ impl HostPathDiscoveryEndpoint {
 
 impl AppServer {
     pub fn new(
-        sk: SSk,
-        pk: SPk,
+        keypair: Option<(SSk, SPk)>,
         addrs: Vec<SocketAddr>,
         verbosity: Verbosity,
         test_helpers: Option<AppServerTest>,
     ) -> anyhow::Result<Self> {
         // setup mio
         let mio_poll = mio::Poll::new()?;
-        let events = mio::Events::with_capacity(20);
+        let events = mio::Events::with_capacity(EVENT_CAPACITY);
         let mut mio_token_dispenser = MioTokenDispenser::default();
 
         // bind each SocketAddr to a socket
@@ -590,22 +618,30 @@ impl AppServer {
         }
 
         // register all sockets to mio
-        for socket in sockets.iter_mut() {
-            mio_poll.registry().register(
-                socket,
-                mio_token_dispenser.dispense(),
-                Interest::READABLE,
-            )?;
+        let mut io_source_index = HashMap::new();
+        for (idx, socket) in sockets.iter_mut().enumerate() {
+            let mio_token = mio_token_dispenser.dispense();
+            mio_poll
+                .registry()
+                .register(socket, mio_token, Interest::READABLE)?;
+            let prev = io_source_index.insert(mio_token, AppServerIoSource::Socket(idx));
+            assert!(prev.is_none());
         }
 
-        // TODO use mio::net::UnixStream together with std::os::unix::net::UnixStream for Linux
+        let crypto_site = match keypair {
+            Some((sk, pk)) => ConstructionSite::from_product(CryptoServer::new(sk, pk)),
+            None => ConstructionSite::new(BuildCryptoServer::empty()),
+        };
 
         Ok(Self {
-            crypt: CryptoServer::new(sk, pk),
+            crypto_site,
             peers: Vec::new(),
             verbosity,
             sockets,
             events,
+            short_poll_queue: Default::default(),
+            performed_long_poll: false,
+            io_source_index,
             mio_poll,
             mio_token_dispenser,
             brokers: BrokerStore::default(),
@@ -616,48 +652,78 @@ impl AppServer {
             unpolled_count: 0,
             last_update_time: Instant::now(),
             test_helpers,
+            #[cfg(feature = "experiment_api")]
+            api_manager: crate::api::mio::MioManager::default(),
         })
     }
 
+    pub fn crypto_server(&self) -> anyhow::Result<&CryptoServer> {
+        self.crypto_site
+            .product_ref()
+            .context("Cryptography handler not initialized")
+    }
+
+    pub fn crypto_server_mut(&mut self) -> anyhow::Result<&mut CryptoServer> {
+        self.crypto_site
+            .product_mut()
+            .context("Cryptography handler not initialized")
+    }
+
     pub fn verbose(&self) -> bool {
         matches!(self.verbosity, Verbosity::Verbose)
     }
 
+    pub fn register_listen_socket(&mut self, mut sock: mio::net::UdpSocket) -> anyhow::Result<()> {
+        let mio_token = self.mio_token_dispenser.dispense();
+        self.mio_poll
+            .registry()
+            .register(&mut sock, mio_token, mio::Interest::READABLE)?;
+        let io_source = self.sockets.len().apply(AppServerIoSource::Socket);
+        self.sockets.push(sock);
+        self.register_io_source(mio_token, io_source);
+        Ok(())
+    }
+
+    pub fn register_io_source(&mut self, token: mio::Token, io_source: AppServerIoSource) {
+        let prev = self.io_source_index.insert(token, io_source);
+        assert!(prev.is_none());
+    }
+
+    pub fn unregister_io_source(&mut self, token: mio::Token) {
+        let value = self.io_source_index.remove(&token);
+        assert!(value.is_some(), "Removed IO source that does not exist");
+    }
+
     pub fn register_broker(
         &mut self,
         broker: Box<dyn WireguardBrokerMio<Error = anyhow::Error, MioError = anyhow::Error>>,
     ) -> Result<BrokerStorePtr> {
         let ptr = Public::from_slice((self.brokers.store.len() as u64).as_bytes());
-
         if self.brokers.store.insert(ptr, broker).is_some() {
             bail!("Broker already registered");
         }
+
+        let mio_token = self.mio_token_dispenser.dispense();
+        let io_source = ptr.apply(AppServerIoSource::PskBroker);
         //Register broker
         self.brokers
             .store
             .get_mut(&ptr)
             .ok_or(anyhow::format_err!("Broker wasn't added to registry"))?
-            .register(
-                self.mio_poll.registry(),
-                self.mio_token_dispenser.dispense(),
-            )?;
+            .register(self.mio_poll.registry(), mio_token)?;
+        self.register_io_source(mio_token, io_source);
 
         Ok(BrokerStorePtr(ptr))
     }
 
     pub fn unregister_broker(&mut self, ptr: BrokerStorePtr) -> Result<()> {
-        //Unregister broker
-        self.brokers
-            .store
-            .get_mut(&ptr.0)
-            .ok_or_else(|| anyhow::anyhow!("Broker not found"))?
-            .unregister(self.mio_poll.registry())?;
-
-        //Remove broker from store
-        self.brokers
+        let mut broker = self
+            .brokers
             .store
             .remove(&ptr.0)
-            .ok_or_else(|| anyhow::anyhow!("Broker not found"))?;
+            .context("Broker not found")?;
+        self.unregister_io_source(broker.mio_token().unwrap());
+        broker.unregister(self.mio_poll.registry())?;
         Ok(())
     }
 
@@ -669,8 +735,13 @@ impl AppServer {
         broker_peer: Option<BrokerPeer>,
         hostname: Option<String>,
     ) -> anyhow::Result<AppPeerPtr> {
-        let PeerPtr(pn) = self.crypt.add_peer(psk, pk)?;
+        let PeerPtr(pn) = match &mut self.crypto_site {
+            ConstructionSite::Void => bail!("Crypto server construction site is void"),
+            ConstructionSite::Builder(builder) => builder.add_peer(psk, pk),
+            ConstructionSite::Product(srv) => srv.add_peer(psk, pk)?,
+        };
         assert!(pn == self.peers.len());
+
         let initial_endpoint = hostname
             .map(Endpoint::discovery_from_hostname)
             .transpose()?;
@@ -712,7 +783,7 @@ impl AppServer {
             );
             if tries_left > 0 {
                 error!("re-initializing networking in {sleep}! {tries_left} tries left.");
-                std::thread::sleep(self.crypt.timebase.dur(sleep));
+                std::thread::sleep(Duration::from_secs_f64(sleep));
                 continue;
             }
 
@@ -755,16 +826,31 @@ impl AppServer {
                 }
             }
 
-            match self.poll(&mut *rx)? {
-                #[allow(clippy::redundant_closure_call)]
-                SendInitiation(peer) => tx_maybe_with!(peer, || self
-                    .crypt
+            enum CryptoSrv {
+                Avail,
+                Missing,
+            }
+
+            let poll_result = self.poll(&mut *rx)?;
+            let have_crypto = match self.crypto_site.is_available() {
+                true => CryptoSrv::Avail,
+                false => CryptoSrv::Missing,
+            };
+
+            #[allow(clippy::redundant_closure_call)]
+            match (have_crypto, poll_result) {
+                (CryptoSrv::Missing, SendInitiation(_)) => {}
+                (CryptoSrv::Avail, SendInitiation(peer)) => tx_maybe_with!(peer, || self
+                    .crypto_server_mut()?
                     .initiate_handshake(peer.lower(), &mut *tx))?,
-                #[allow(clippy::redundant_closure_call)]
-                SendRetransmission(peer) => tx_maybe_with!(peer, || self
-                    .crypt
+
+                (CryptoSrv::Missing, SendRetransmission(_)) => {}
+                (CryptoSrv::Avail, SendRetransmission(peer)) => tx_maybe_with!(peer, || self
+                    .crypto_server_mut()?
                     .retransmit_handshake(peer.lower(), &mut *tx))?,
-                DeleteKey(peer) => {
+
+                (CryptoSrv::Missing, DeleteKey(_)) => {}
+                (CryptoSrv::Avail, DeleteKey(peer)) => {
                     self.output_key(peer, Stale, &SymKey::random())?;
 
                     // There was a loss of connection apparently; restart host discovery
@@ -778,12 +864,15 @@ impl AppServer {
                     );
                 }
 
-                ReceivedMessage(len, endpoint) => {
+                (CryptoSrv::Missing, ReceivedMessage(_, _)) => {}
+                (CryptoSrv::Avail, ReceivedMessage(len, endpoint)) => {
                     let msg_result = match self.under_load {
                         DoSOperation::UnderLoad => {
                             self.handle_msg_under_load(&endpoint, &rx[..len], &mut *tx)
                         }
-                        DoSOperation::Normal => self.crypt.handle_msg(&rx[..len], &mut *tx),
+                        DoSOperation::Normal => {
+                            self.crypto_server_mut()?.handle_msg(&rx[..len], &mut *tx)
+                        }
                     };
                     match msg_result {
                         Err(ref e) => {
@@ -811,7 +900,8 @@ impl AppServer {
                                 ap.get_app_mut(self).current_endpoint = Some(endpoint);
 
                                 // TODO: Maybe we should rather call the key "rosenpass output"?
-                                self.output_key(ap, Exchanged, &self.crypt.osk(p)?)?;
+                                let osk = &self.crypto_server_mut()?.osk(p)?;
+                                self.output_key(ap, Exchanged, osk)?;
                             }
                         }
                     }
@@ -827,9 +917,9 @@ impl AppServer {
         tx: &mut [u8],
     ) -> Result<crate::protocol::HandleMsgResult> {
         match endpoint {
-            Endpoint::SocketBoundAddress(socket) => {
-                self.crypt.handle_msg_under_load(rx, &mut *tx, socket)
-            }
+            Endpoint::SocketBoundAddress(socket) => self
+                .crypto_server_mut()?
+                .handle_msg_under_load(rx, &mut *tx, socket),
             Endpoint::Discovery(_) => {
                 anyhow::bail!("Host-path discovery is not supported under load")
             }
@@ -842,7 +932,7 @@ impl AppServer {
         why: KeyOutputReason,
         key: &SymKey,
     ) -> anyhow::Result<()> {
-        let peerid = peer.lower().get(&self.crypt).pidt()?;
+        let peerid = peer.lower().get(self.crypto_server()?).pidt()?;
 
         if self.verbose() {
             let msg = match why {
@@ -870,10 +960,14 @@ impl AppServer {
 
             // this is intentionally writing to stdout instead of stderr, because
             // it is meant to allow external detection of a successful key-exchange
-            println!(
+            let stdout = stdout();
+            let mut stdout = stdout.lock();
+            writeln!(
+                stdout,
                 "output-key peer {} key-file {of:?} {why}",
                 peerid.fmt_b64::<MAX_B64_PEER_ID_SIZE>()
-            );
+            )?;
+            stdout.flush()?;
         }
 
         peer.set_psk(self, key)?;
@@ -884,17 +978,32 @@ impl AppServer {
     pub fn poll(&mut self, rx_buf: &mut [u8]) -> anyhow::Result<AppPollResult> {
         use crate::protocol::PollResult as C;
         use AppPollResult as A;
-        loop {
-            return Ok(match self.crypt.poll()? {
-                C::DeleteKey(PeerPtr(no)) => A::DeleteKey(AppPeerPtr(no)),
-                C::SendInitiation(PeerPtr(no)) => A::SendInitiation(AppPeerPtr(no)),
-                C::SendRetransmission(PeerPtr(no)) => A::SendRetransmission(AppPeerPtr(no)),
-                C::Sleep(timeout) => match self.try_recv(rx_buf, timeout)? {
-                    Some((len, addr)) => A::ReceivedMessage(len, addr),
-                    None => continue,
-                },
-            });
-        }
+        let res = loop {
+            // Call CryptoServer's poll (if available)
+            let crypto_poll = self
+                .crypto_site
+                .product_mut()
+                .map(|crypto| crypto.poll())
+                .transpose()?;
+
+            // Map crypto server's poll result to our poll result
+            let io_poll_timeout = match crypto_poll {
+                Some(C::DeleteKey(PeerPtr(no))) => break A::DeleteKey(AppPeerPtr(no)),
+                Some(C::SendInitiation(PeerPtr(no))) => break A::SendInitiation(AppPeerPtr(no)),
+                Some(C::SendRetransmission(PeerPtr(no))) => {
+                    break A::SendRetransmission(AppPeerPtr(no))
+                }
+                Some(C::Sleep(timeout)) => timeout, // No event from crypto-server, do IO
+                None => crate::protocol::UNENDING,  // Crypto server is uninitialized, do IO
+            };
+
+            // Perform IO (look for a message)
+            if let Some((len, addr)) = self.try_recv(rx_buf, io_poll_timeout)? {
+                break A::ReceivedMessage(len, addr);
+            }
+        };
+
+        Ok(res)
     }
 
     /// Tries to receive a new message
@@ -932,22 +1041,33 @@ impl AppServer {
         // readiness event seems to be good enough™ for now.
 
         // only poll if we drained all sockets before
-        if self.all_sockets_drained {
-            //Non blocked polling
-            self.mio_poll
-                .poll(&mut self.events, Some(Duration::from_secs(0)))?;
-
-            if self.events.iter().peekable().peek().is_none() {
-                // if there are no events, then add to blocking poll count
-                self.blocking_polls_count += 1;
-                //Execute blocking poll
-                self.mio_poll.poll(&mut self.events, Some(timeout))?;
-            } else {
-                self.non_blocking_polls_count += 1;
+        run(|| -> anyhow::Result<()> {
+            if !self.all_sockets_drained || !self.short_poll_queue.is_empty() {
+                self.unpolled_count += 1;
+                return Ok(());
             }
-        } else {
-            self.unpolled_count += 1;
-        }
+
+            self.perform_mio_poll_and_register_events(Duration::from_secs(0))?; // Non-blocking poll
+            if !self.short_poll_queue.is_empty() {
+                // Got some events in non-blocking mode
+                self.non_blocking_polls_count += 1;
+                return Ok(());
+            }
+
+            if !self.performed_long_poll {
+                // pass – go perform a full long poll before we enter blocking poll mode
+                // to make sure our experimental short poll feature did not miss any events
+                // due to being buggy.
+                return Ok(());
+            }
+
+            // Perform and register blocking poll
+            self.blocking_polls_count += 1;
+            self.perform_mio_poll_and_register_events(timeout)?;
+            self.performed_long_poll = false;
+
+            Ok(())
+        })?;
 
         if let Some(AppServerTest {
             enable_dos_permanently: true,
@@ -982,26 +1102,58 @@ impl AppServer {
             }
         }
 
+        // Focused polling – i.e. actually using mio::Token – is experimental for now.
+        // The reason for this is that we need to figure out how to integrate load detection
+        // and focused polling for one. Mio event-based polling also does not play nice with
+        // the current function signature and its reentrant design which is focused around receiving UDP socket packages
+        // for processing by the crypto protocol server.
+        // Besides that, there are also some parts of the code which intentionally block
+        // despite available data. This is the correct behavior; e.g. api::mio::Connection blocks
+        // further reads from its unix socket until the write buffer is flushed. In other words
+        // the connection handler makes sure that there is a buffer to put the response in while
+        // before reading further request.
+        // The potential problem with this behavior is that we end up ignoring instructions from
+        // epoll() to read from the particular sockets, so epoll will return information about that
+        // particular – blocked – file descriptor every call. We have only so many event slots and
+        // in theory, the event array could fill up entirely with intentionally blocked sockets.
+        // We need to figure out how to deal with this situation.
+        // Mio uses uses epoll in level-triggered mode, so we could handle taint-tracking for ignored
+        // sockets ourselves. The facilities are available in epoll and Mio, but we need to figure out how mio uses those
+        // facilities and how we can integrate them here.
+        // This will involve rewriting a lot of IO code and we should probably have integration
+        // tests  before we approach that.
+        //
+        // This hybrid approach is not without merit though; the short poll implementation covers
+        // all our IO sources, so under contention, rosenpass should generally not hit the long
+        // poll mode below. We keep short polling and calling epoll() in non-blocking mode (timeout
+        // of zero) until we run out of IO events processed. Then, just before we would perform a
+        // blocking poll, we go through all available IO sources to see if we missed anything.
+        {
+            while let Some(ev) = self.short_poll_queue.pop_front() {
+                if let Some(v) = self.try_recv_from_mio_token(buf, ev.token())? {
+                    return Ok(Some(v));
+                }
+            }
+        }
+
         // drain all sockets
         let mut would_block_count = 0;
-        for (sock_no, socket) in self.sockets.iter_mut().enumerate() {
-            match socket.recv_from(buf) {
-                Ok((n, addr)) => {
+        for sock_no in 0..self.sockets.len() {
+            match self
+                .try_recv_from_listen_socket(buf, sock_no)
+                .io_err_kind_hint()
+            {
+                Ok(None) => continue,
+                Ok(Some(v)) => {
                     // at least one socket was not drained...
                     self.all_sockets_drained = false;
-                    return Ok(Some((
-                        n,
-                        Endpoint::SocketBoundAddress(SocketBoundEndpoint::new(
-                            SocketPtr(sock_no),
-                            addr,
-                        )),
-                    )));
+                    return Ok(Some(v));
                 }
-                Err(e) if e.kind() == ErrorKind::WouldBlock => {
+                Err((_, ErrorKind::WouldBlock)) => {
                     would_block_count += 1;
                 }
                 // TODO if one socket continuously returns an error, then we never poll, thus we never wait for a timeout, thus we have a spin-lock
-                Err(e) => return Err(e.into()),
+                Err((e, _)) => return Err(e)?,
             }
         }
 
@@ -1013,6 +1165,126 @@ impl AppServer {
             broker.process_poll()?;
         }
 
+        // API poll
+
+        #[cfg(feature = "experiment_api")]
+        {
+            use crate::api::mio::MioManagerContext;
+            MioManagerFocus(self).poll()?;
+        }
+
+        self.performed_long_poll = true;
+
         Ok(None)
     }
+
+    fn perform_mio_poll_and_register_events(&mut self, timeout: Duration) -> io::Result<()> {
+        self.mio_poll.poll(&mut self.events, Some(timeout))?;
+        // Fill the short poll buffer with the acquired events
+        self.events
+            .iter()
+            .cloned()
+            .for_each(|v| self.short_poll_queue.push_back(v));
+        Ok(())
+    }
+
+    fn try_recv_from_mio_token(
+        &mut self,
+        buf: &mut [u8],
+        token: mio::Token,
+    ) -> anyhow::Result<Option<(usize, Endpoint)>> {
+        let io_source = match self.io_source_index.get(&token) {
+            Some(io_source) => *io_source,
+            None => {
+                log::warn!("No IO source assiociated with mio token ({token:?}). Polling using mio tokens directly is an experimental feature and IO handler should recover when all available io sources are polled. This is a developer error. Please report it.");
+                return Ok(None);
+            }
+        };
+
+        self.try_recv_from_io_source(buf, io_source)
+    }
+
+    fn try_recv_from_io_source(
+        &mut self,
+        buf: &mut [u8],
+        io_source: AppServerIoSource,
+    ) -> anyhow::Result<Option<(usize, Endpoint)>> {
+        use crate::api::mio::MioManagerContext;
+
+        match io_source {
+            AppServerIoSource::Socket(idx) => self
+                .try_recv_from_listen_socket(buf, idx)
+                .substitute_for_ioerr_wouldblock(None)?
+                .ok(),
+
+            #[cfg(feature = "experiment_api")]
+            AppServerIoSource::PskBroker(key) => self
+                .brokers
+                .store
+                .get_mut(&key)
+                .with_context(|| format!("No PSK broker under key {key:?}"))?
+                .process_poll()
+                .map(|_| None),
+
+            #[cfg(feature = "experiment_api")]
+            AppServerIoSource::MioManager(mmio_src) => MioManagerFocus(self)
+                .poll_particular(mmio_src)
+                .map(|_| None),
+        }
+    }
+
+    fn try_recv_from_listen_socket(
+        &mut self,
+        buf: &mut [u8],
+        idx: usize,
+    ) -> io::Result<Option<(usize, Endpoint)>> {
+        use std::io::ErrorKind as K;
+        let (n, addr) = loop {
+            match self.sockets[idx].recv_from(buf).io_err_kind_hint() {
+                Ok(v) => break v,
+                Err((_, K::Interrupted)) => continue,
+                Err((e, _)) => return Err(e)?,
+            }
+        };
+        SocketPtr(idx)
+            .apply(|sp| SocketBoundEndpoint::new(sp, addr))
+            .apply(Endpoint::SocketBoundAddress)
+            .apply(|ep| (n, ep))
+            .some()
+            .ok()
+    }
+
+    #[cfg(feature = "experiment_api")]
+    pub fn add_api_connection(&mut self, connection: mio::net::UnixStream) -> std::io::Result<()> {
+        use crate::api::mio::MioManagerContext;
+        MioManagerFocus(self).add_connection(connection)
+    }
+
+    #[cfg(feature = "experiment_api")]
+    pub fn add_api_listener(&mut self, listener: mio::net::UnixListener) -> std::io::Result<()> {
+        use crate::api::mio::MioManagerContext;
+        MioManagerFocus(self).add_listener(listener)
+    }
+}
+
+#[cfg(feature = "experiment_api")]
+struct MioManagerFocus<'a>(&'a mut AppServer);
+
+#[cfg(feature = "experiment_api")]
+impl crate::api::mio::MioManagerContext for MioManagerFocus<'_> {
+    fn mio_manager(&self) -> &crate::api::mio::MioManager {
+        &self.0.api_manager
+    }
+
+    fn mio_manager_mut(&mut self) -> &mut crate::api::mio::MioManager {
+        &mut self.0.api_manager
+    }
+
+    fn app_server(&self) -> &AppServer {
+        self.0
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        self.0
+    }
 }

rosenpass/src/bin/gen-ipc-msg-types.rs

@@ -0,0 +1,92 @@
+use anyhow::{Context, Result};
+use heck::ToShoutySnakeCase;
+
+use rosenpass_ciphers::{hash_domain::HashDomain, KEY_LEN};
+
+fn calculate_hash_value(hd: HashDomain, values: &[&str]) -> Result<[u8; KEY_LEN]> {
+    match values.split_first() {
+        Some((head, tail)) => calculate_hash_value(hd.mix(head.as_bytes())?, tail),
+        None => Ok(hd.into_value()),
+    }
+}
+
+fn print_literal(path: &[&str]) -> Result<()> {
+    let val = calculate_hash_value(HashDomain::zero(), path)?;
+    let (last, prefix) = path.split_last().context("developer error!")?;
+    let var_name = last.to_shouty_snake_case();
+
+    print!("// hash domain hash of: ");
+    for n in prefix.iter() {
+        print!("{n} -> ");
+    }
+    println!("{last}");
+
+    let c = hex::encode(val)
+        .chars()
+        .collect::<Vec<char>>()
+        .chunks_exact(4)
+        .map(|chunk| chunk.iter().collect::<String>())
+        .collect::<Vec<_>>();
+    println!("const {var_name} : RawMsgType = RawMsgType::from_le_bytes(hex!(\"{} {} {} {}    {} {} {} {}\"));",
+        c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7]);
+
+    Ok(())
+}
+
+#[derive(Debug, Clone)]
+enum Tree {
+    Branch(String, Vec<Tree>),
+    Leaf(String),
+}
+
+impl Tree {
+    fn name(&self) -> &str {
+        match self {
+            Self::Branch(name, _) => name,
+            Self::Leaf(name) => name,
+        }
+    }
+
+    fn gen_code_inner(&self, prefix: &[&str]) -> Result<()> {
+        let mut path = prefix.to_owned();
+        path.push(self.name());
+
+        match self {
+            Self::Branch(_, ref children) => {
+                for c in children.iter() {
+                    c.gen_code_inner(&path)?
+                }
+            }
+            Self::Leaf(_) => print_literal(&path)?,
+        };
+
+        Ok(())
+    }
+
+    fn gen_code(&self) -> Result<()> {
+        self.gen_code_inner(&[])
+    }
+}
+
+fn main() -> Result<()> {
+    let tree = Tree::Branch(
+        "Rosenpass IPC API".to_owned(),
+        vec![Tree::Branch(
+            "Rosenpass Protocol Server".to_owned(),
+            vec![
+                Tree::Leaf("Ping Request".to_owned()),
+                Tree::Leaf("Ping Response".to_owned()),
+                Tree::Leaf("Supply Keypair Request".to_owned()),
+                Tree::Leaf("Supply Keypair Response".to_owned()),
+                Tree::Leaf("Add Listen Socket Request".to_owned()),
+                Tree::Leaf("Add Listen Socket Response".to_owned()),
+                Tree::Leaf("Add Psk Broker Request".to_owned()),
+                Tree::Leaf("Add Psk Broker Response".to_owned()),
+            ],
+        )],
+    );
+
+    println!("type RawMsgType = u128;");
+    println!();
+    tree.gen_code()
+}

rosenpass/src/cli.rs

@@ -1,15 +1,13 @@
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Context};
 use clap::{Parser, Subcommand};
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
 use rosenpass_secret_memory::file::StoreSecret;
-use rosenpass_secret_memory::{
-    secret_policy_try_use_memfd_secrets, secret_policy_use_only_malloc_secrets,
-};
-use rosenpass_util::file::{LoadValue, LoadValueB64};
+use rosenpass_util::file::{LoadValue, LoadValueB64, StoreValue};
 use rosenpass_wireguard_broker::brokers::native_unix::{
     NativeUnixBroker, NativeUnixBrokerConfigBaseBuilder, NativeUnixBrokerConfigBaseBuilderError,
 };
+use std::ops::DerefMut;
 use std::path::PathBuf;
 
 use crate::app_server::AppServerTest;
@@ -18,6 +16,29 @@ use crate::protocol::{SPk, SSk, SymKey};
 
 use super::config;
 
+#[cfg(feature = "experiment_api")]
+use {
+    command_fds::{CommandFdExt, FdMapping},
+    log::{error, info},
+    mio::net::UnixStream,
+    rosenpass_util::fd::claim_fd,
+    rosenpass_wireguard_broker::brokers::mio_client::MioBrokerClient,
+    rosenpass_wireguard_broker::WireguardBrokerMio,
+    rustix::fd::AsRawFd,
+    rustix::net::{socketpair, AddressFamily, SocketFlags, SocketType},
+    std::os::unix::net,
+    std::process::Command,
+    std::thread,
+};
+
+/// enum representing a choice of interface to a WireGuard broker
+#[derive(Debug)]
+pub enum BrokerInterface {
+    Socket(PathBuf),
+    FileDescriptor(i32),
+    SocketPair,
+}
+
 /// struct holding all CLI arguments for `clap` crate to parse
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about)]
@@ -34,11 +55,41 @@ pub struct CliArgs {
     #[arg(short, long, group = "log-level")]
     quiet: bool,
 
+    #[command(flatten)]
+    #[cfg(feature = "experiment_api")]
+    api: crate::api::cli::ApiCli,
+
+    /// path of the wireguard_psk broker socket to connect to
+    #[cfg(feature = "experiment_api")]
+    #[arg(long, group = "psk-broker-specs")]
+    psk_broker_path: Option<PathBuf>,
+
+    /// fd of the wireguard_spk broker socket to connect to
+    ///
+    /// when this command is called from another process, the other process can open and bind the
+    /// Unix socket for the psk broker connection to use themselves, passing it to this process --
+    /// in Rust this can be achieved using the
+    /// [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate
+    #[cfg(feature = "experiment_api")]
+    #[arg(long, group = "psk-broker-specs")]
+    psk_broker_fd: Option<i32>,
+
+    /// spawn a psk broker locally using a socket pair
+    #[cfg(feature = "experiment_api")]
+    #[arg(short, long, group = "psk-broker-specs")]
+    psk_broker_spawn: bool,
+
     #[command(subcommand)]
     pub command: CliCommand,
 }
 
 impl CliArgs {
+    pub fn apply_to_config(&self, _cfg: &mut config::Rosenpass) -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        self.api.apply_to_config(_cfg)?;
+        Ok(())
+    }
+
     /// returns the log level filter set by CLI args
     /// returns `None` if the user did not specify any log level filter via CLI
     ///
@@ -50,13 +101,35 @@ impl CliArgs {
             return Some(log::LevelFilter::Info);
         }
         if self.quiet {
-            return Some(log::LevelFilter::Error);
+            return Some(log::LevelFilter::Warn);
         }
         if let Some(level_filter) = self.log_level {
             return Some(level_filter);
         }
         None
     }
+
+    #[cfg(feature = "experiment_api")]
+    /// returns the broker interface set by CLI args
+    /// returns `None` if the `experiment_api` feature isn't enabled
+    pub fn get_broker_interface(&self) -> Option<BrokerInterface> {
+        if let Some(path_ref) = self.psk_broker_path.as_ref() {
+            Some(BrokerInterface::Socket(path_ref.to_path_buf()))
+        } else if let Some(fd) = self.psk_broker_fd {
+            Some(BrokerInterface::FileDescriptor(fd))
+        } else if self.psk_broker_spawn {
+            Some(BrokerInterface::SocketPair)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(not(feature = "experiment_api"))]
+    /// returns the broker interface set by CLI args
+    /// returns `None` if the `experiment_api` feature isn't enabled
+    pub fn get_broker_interface(&self) -> Option<BrokerInterface> {
+        None
+    }
 }
 
 /// represents a command specified via CLI
@@ -151,21 +224,18 @@ pub enum CliCommand {
     Man,
 }
 
-impl CliCommand {
+impl CliArgs {
     /// runs the command specified via CLI
     ///
     /// ## TODO
     /// - This method consumes the [`CliCommand`] value. It might be wise to use a reference...
-    pub fn run(self, test_helpers: Option<AppServerTest>) -> anyhow::Result<()> {
-        //Specify secret policy
-
-        #[cfg(feature = "enable_memfd_alloc")]
-        secret_policy_try_use_memfd_secrets();
-        #[cfg(not(feature = "enable_memfd_alloc"))]
-        secret_policy_use_only_malloc_secrets();
-
+    pub fn run(
+        self,
+        broker_interface: Option<BrokerInterface>,
+        test_helpers: Option<AppServerTest>,
+    ) -> anyhow::Result<()> {
         use CliCommand::*;
-        match self {
+        match &self.command {
             Man => {
                 let man_cmd = std::process::Command::new("man")
                     .args(["1", "rosenpass"])
@@ -177,7 +247,7 @@ impl CliCommand {
             }
             GenConfig { config_file, force } => {
                 ensure!(
-                    force || !config_file.exists(),
+                    *force || !config_file.exists(),
                     "config file {config_file:?} already exists"
                 );
 
@@ -192,9 +262,9 @@ impl CliCommand {
                 let mut secret_key: Option<PathBuf> = None;
 
                 // Manual arg parsing, since clap wants to prefix flags with "--"
-                let mut args = args.into_iter();
+                let mut args = args.iter();
                 loop {
-                    match (args.next().as_deref(), args.next()) {
+                    match (args.next().map(|x| x.as_str()), args.next()) {
                         (Some("private-key"), Some(opt)) | (Some("secret-key"), Some(opt)) => {
                             secret_key = Some(opt.into());
                         }
@@ -233,10 +303,13 @@ impl CliCommand {
                         );
 
                         let config = config::Rosenpass::load(config_file)?;
+                        let keypair = config
+                            .keypair
+                            .context("Config file present, but no keypair is specified.")?;
 
-                        (config.public_key, config.secret_key)
+                        (keypair.public_key, keypair.secret_key)
                     }
-                    (_, Some(pkf), Some(skf)) => (pkf, skf),
+                    (_, Some(pkf), Some(skf)) => (pkf.clone(), skf.clone()),
                     _ => {
                         bail!("either a config-file or both public-key and secret-key file are required")
                     }
@@ -246,12 +319,14 @@ impl CliCommand {
                 let mut problems = vec![];
                 if !force && pkf.is_file() {
                     problems.push(format!(
-                        "public-key file {pkf:?} exist, refusing to overwrite it"
+                        "public-key file {:?} exists, refusing to overwrite",
+                        std::fs::canonicalize(&pkf)?,
                     ));
                 }
                 if !force && skf.is_file() {
                     problems.push(format!(
-                        "secret-key file {skf:?} exist, refusing to overwrite it"
+                        "secret-key file {:?} exists, refusing to overwrite",
+                        std::fs::canonicalize(&skf)?,
                     ));
                 }
                 if !problems.is_empty() {
@@ -268,31 +343,38 @@ impl CliCommand {
                     "config file '{config_file:?}' does not exist"
                 );
 
-                let config = config::Rosenpass::load(config_file)?;
+                let mut config = config::Rosenpass::load(config_file)?;
                 config.validate()?;
-                Self::event_loop(config, test_helpers)?;
+                self.apply_to_config(&mut config)?;
+                config.check_usefullness()?;
+
+                Self::event_loop(config, broker_interface, test_helpers)?;
             }
 
             Exchange {
                 first_arg,
-                mut rest_of_args,
+                rest_of_args,
                 config_file,
             } => {
-                rest_of_args.insert(0, first_arg);
+                let mut rest_of_args = rest_of_args.clone();
+                rest_of_args.insert(0, first_arg.clone());
                 let args = rest_of_args;
                 let mut config = config::Rosenpass::parse_args(args)?;
 
                 if let Some(p) = config_file {
-                    config.store(&p)?;
-                    config.config_file_path = p;
+                    config.store(p)?;
+                    config.config_file_path.clone_from(p);
                 }
                 config.validate()?;
-                Self::event_loop(config, test_helpers)?;
+                self.apply_to_config(&mut config)?;
+                config.check_usefullness()?;
+
+                Self::event_loop(config, broker_interface, test_helpers)?;
             }
 
             Validate { config_files } => {
                 for file in config_files {
-                    match config::Rosenpass::load(&file) {
+                    match config::Rosenpass::load(file) {
                         Ok(config) => {
                             eprintln!("{file:?} is valid TOML and conforms to the expected schema");
                             match config.validate() {
@@ -311,23 +393,34 @@ impl CliCommand {
 
     fn event_loop(
         config: config::Rosenpass,
+        broker_interface: Option<BrokerInterface>,
         test_helpers: Option<AppServerTest>,
     ) -> anyhow::Result<()> {
         const MAX_PSK_SIZE: usize = 1000;
+
         // load own keys
-        let sk = SSk::load(&config.secret_key)?;
-        let pk = SPk::load(&config.public_key)?;
+        let keypair = config
+            .keypair
+            .as_ref()
+            .map(|kp| -> anyhow::Result<_> {
+                let sk = SSk::load(&kp.secret_key)?;
+                let pk = SPk::load(&kp.public_key)?;
+                Ok((sk, pk))
+            })
+            .transpose()?;
 
         // start an application server
         let mut srv = std::boxed::Box::<AppServer>::new(AppServer::new(
-            sk,
-            pk,
-            config.listen,
+            keypair,
+            config.listen.clone(),
             config.verbosity,
             test_helpers,
         )?);
 
-        let broker_store_ptr = srv.register_broker(Box::new(NativeUnixBroker::new()))?;
+        config.apply_to_app_server(&mut srv)?;
+
+        let broker = Self::create_broker(broker_interface)?;
+        let broker_store_ptr = srv.register_broker(broker)?;
 
         fn cfg_err_map(e: NativeUnixBrokerConfigBaseBuilderError) -> anyhow::Error {
             anyhow::Error::msg(format!("NativeUnixBrokerConfigBaseBuilderError: {:?}", e))
@@ -364,13 +457,102 @@ impl CliCommand {
 
         srv.event_loop()
     }
+
+    #[cfg(feature = "experiment_api")]
+    fn create_broker(
+        broker_interface: Option<BrokerInterface>,
+    ) -> Result<
+        Box<dyn WireguardBrokerMio<MioError = anyhow::Error, Error = anyhow::Error>>,
+        anyhow::Error,
+    > {
+        if let Some(interface) = broker_interface {
+            let socket = Self::get_broker_socket(interface)?;
+            Ok(Box::new(MioBrokerClient::new(socket)))
+        } else {
+            Ok(Box::new(NativeUnixBroker::new()))
+        }
+    }
+
+    #[cfg(not(feature = "experiment_api"))]
+    fn create_broker(
+        _broker_interface: Option<BrokerInterface>,
+    ) -> Result<Box<NativeUnixBroker>, anyhow::Error> {
+        Ok(Box::new(NativeUnixBroker::new()))
+    }
+
+    #[cfg(feature = "experiment_api")]
+    fn get_broker_socket(broker_interface: BrokerInterface) -> Result<UnixStream, anyhow::Error> {
+        // Connect to the psk broker unix socket if one was specified
+        // OR OTHERWISE spawn the psk broker and use socketpair(2) to connect with them
+        match broker_interface {
+            BrokerInterface::Socket(broker_path) => Ok(UnixStream::connect(broker_path)?),
+            BrokerInterface::FileDescriptor(broker_fd) => {
+                // mio::net::UnixStream doesn't implement From<OwnedFd>, so we have to go through std
+                let sock = net::UnixStream::from(claim_fd(broker_fd)?);
+                sock.set_nonblocking(true)?;
+                Ok(UnixStream::from_std(sock))
+            }
+            BrokerInterface::SocketPair => {
+                // Form a socketpair for communicating to the broker
+                let (ours, theirs) = socketpair(
+                    AddressFamily::UNIX,
+                    SocketType::STREAM,
+                    SocketFlags::empty(),
+                    None,
+                )?;
+
+                // Setup our end of the socketpair
+                let ours = net::UnixStream::from(ours);
+                ours.set_nonblocking(true)?;
+
+                // Start the PSK broker
+                let mut child = Command::new("rosenpass-wireguard-broker-socket-handler")
+                    .args(["--stream-fd", "3"])
+                    .fd_mappings(vec![FdMapping {
+                        parent_fd: theirs.as_raw_fd(),
+                        child_fd: 3,
+                    }])?
+                    .spawn()?;
+
+                // Handle the PSK broker crashing
+                thread::spawn(move || {
+                    let status = child.wait();
+
+                    if let Ok(status) = status {
+                        if status.success() {
+                            // Maybe they are doing double forking?
+                            info!("PSK broker exited.");
+                        } else {
+                            error!("PSK broker exited with an error ({status:?})");
+                        }
+                    } else {
+                        error!("Wait on PSK broker process failed ({status:?})");
+                    }
+                });
+
+                Ok(UnixStream::from_std(ours))
+            }
+        }
+    }
 }
 
 /// generate secret and public keys, store in files according to the paths passed as arguments
 fn generate_and_save_keypair(secret_key: PathBuf, public_key: PathBuf) -> anyhow::Result<()> {
     let mut ssk = crate::protocol::SSk::random();
     let mut spk = crate::protocol::SPk::random();
-    StaticKem::keygen(ssk.secret_mut(), spk.secret_mut())?;
+    StaticKem::keygen(ssk.secret_mut(), spk.deref_mut())?;
     ssk.store_secret(secret_key)?;
     spk.store(public_key)
 }
+
+#[cfg(feature = "internal_testing")]
+pub mod testing {
+    use super::*;
+
+    pub fn generate_and_save_keypair(
+        secret_key: PathBuf,
+        public_key: PathBuf,
+    ) -> anyhow::Result<()> {
+        super::generate_and_save_keypair(secret_key, public_key)
+    }
+}

rosenpass/src/config.rs

@@ -19,13 +19,28 @@ use anyhow::{bail, ensure};
 use rosenpass_util::file::{fopen_w, Visibility};
 use serde::{Deserialize, Serialize};
 
+use crate::app_server::AppServer;
+
+#[cfg(feature = "experiment_api")]
+fn empty_api_config() -> crate::api::config::ApiConfig {
+    crate::api::config::ApiConfig {
+        listen_path: Vec::new(),
+        listen_fd: Vec::new(),
+        stream_fd: Vec::new(),
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Rosenpass {
-    /// path to the public key file
-    pub public_key: PathBuf,
+    // TODO: Raise error if secret key or public key alone is set during deserialization
+    // SEE: https://github.com/serde-rs/serde/issues/2793
+    #[serde(flatten)]
+    pub keypair: Option<Keypair>,
 
-    /// path to the secret key file
-    pub secret_key: PathBuf,
+    /// Location of the API listen sockets
+    #[cfg(feature = "experiment_api")]
+    #[serde(default = "empty_api_config")]
+    pub api: crate::api::config::ApiConfig,
 
     /// list of [`SocketAddr`] to listen on
     ///
@@ -52,9 +67,29 @@ pub struct Rosenpass {
     pub config_file_path: PathBuf,
 }
 
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)]
+pub struct Keypair {
+    /// path to the public key file
+    pub public_key: PathBuf,
+
+    /// path to the secret key file
+    pub secret_key: PathBuf,
+}
+
+impl Keypair {
+    pub fn new<Pk: AsRef<Path>, Sk: AsRef<Path>>(public_key: Pk, secret_key: Sk) -> Self {
+        let public_key = public_key.as_ref().to_path_buf();
+        let secret_key = secret_key.as_ref().to_path_buf();
+        Self {
+            public_key,
+            secret_key,
+        }
+    }
+}
+
 /// ## TODO
 /// - replace this type with [`log::LevelFilter`], also see <https://github.com/rosenpass/rosenpass/pull/246>
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Copy, Clone)]
 pub enum Verbosity {
     Quiet,
     Verbose,
@@ -107,6 +142,12 @@ pub struct WireGuard {
     pub extra_params: Vec<String>,
 }
 
+impl Default for Rosenpass {
+    fn default() -> Self {
+        Self::empty()
+    }
+}
+
 impl Rosenpass {
     /// load configuration from a TOML file
     ///
@@ -122,8 +163,10 @@ impl Rosenpass {
 
         // resolve `~` (see https://github.com/rosenpass/rosenpass/issues/237)
         use util::resolve_path_with_tilde;
-        resolve_path_with_tilde(&mut config.public_key);
-        resolve_path_with_tilde(&mut config.secret_key);
+        if let Some(ref mut keypair) = config.keypair {
+            resolve_path_with_tilde(&mut keypair.public_key);
+            resolve_path_with_tilde(&mut keypair.secret_key);
+        }
         for peer in config.peers.iter_mut() {
             resolve_path_with_tilde(&mut peer.public_key);
             if let Some(ref mut psk) = &mut peer.pre_shared_key {
@@ -157,25 +200,33 @@ impl Rosenpass {
         self.store(&self.config_file_path)
     }
 
+    pub fn apply_to_app_server(&self, _srv: &mut AppServer) -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        self.api.apply_to_app_server(_srv)?;
+        Ok(())
+    }
+
     /// Validate a configuration
     ///
     /// ## TODO
     /// - check that files do not just exist but are also readable
     /// - warn if neither out_key nor exchange_command of a peer is defined (v.i.)
     pub fn validate(&self) -> anyhow::Result<()> {
-        // check the public key file exists
-        ensure!(
-            self.public_key.is_file(),
-            "could not find public-key file {:?}: no such file",
-            self.public_key
-        );
+        if let Some(ref keypair) = self.keypair {
+            // check the public key file exists
+            ensure!(
+                keypair.public_key.is_file(),
+                "could not find public-key file {:?}: no such file",
+                keypair.public_key
+            );
 
-        // check the secret-key file exists
-        ensure!(
-            self.secret_key.is_file(),
-            "could not find secret-key file {:?}: no such file",
-            self.secret_key
-        );
+            // check the secret-key file exists
+            ensure!(
+                keypair.secret_key.is_file(),
+                "could not find secret-key file {:?}: no such file",
+                keypair.secret_key
+            );
+        }
 
         for (i, peer) in self.peers.iter().enumerate() {
             // check peer's public-key file exists
@@ -200,12 +251,36 @@ impl Rosenpass {
         Ok(())
     }
 
+    pub fn check_usefullness(&self) -> anyhow::Result<()> {
+        #[cfg(not(feature = "experiment_api"))]
+        ensure!(self.keypair.is_some(), "Server keypair missing.");
+
+        #[cfg(feature = "experiment_api")]
+        ensure!(
+            self.keypair.is_some() || self.api.has_api_sources(),
+            "{}{}",
+            "Specify a server keypair or some API connections to configure the keypair with.",
+            "Without a keypair, rosenpass can not operate."
+        );
+
+        Ok(())
+    }
+
+    pub fn empty() -> Self {
+        Self::new(None)
+    }
+
+    pub fn from_sk_pk<Sk: AsRef<Path>, Pk: AsRef<Path>>(sk: Sk, pk: Pk) -> Self {
+        Self::new(Some(Keypair::new(pk, sk)))
+    }
+
     /// Creates a new configuration
-    pub fn new<P1: AsRef<Path>, P2: AsRef<Path>>(public_key: P1, secret_key: P2) -> Self {
+    pub fn new(keypair: Option<Keypair>) -> Self {
         Self {
-            public_key: PathBuf::from(public_key.as_ref()),
-            secret_key: PathBuf::from(secret_key.as_ref()),
+            keypair,
             listen: vec![],
+            #[cfg(feature = "experiment_api")]
+            api: crate::api::config::ApiConfig::default(),
             verbosity: Verbosity::Quiet,
             peers: vec![],
             config_file_path: PathBuf::new(),
@@ -228,7 +303,7 @@ impl Rosenpass {
     /// from chaotic args
     /// Quest: the grammar is undecideable, what do we do here?
     pub fn parse_args(args: Vec<String>) -> anyhow::Result<Self> {
-        let mut config = Self::new("", "");
+        let mut config = Self::new(Some(Keypair::new("", "")));
 
         #[derive(Debug, Hash, PartialEq, Eq)]
         enum State {
@@ -289,7 +364,7 @@ impl Rosenpass {
                         already_set.insert(OwnPublicKey),
                         "public-key was already set"
                     );
-                    config.public_key = pk.into();
+                    config.keypair.as_mut().unwrap().public_key = pk.into();
                     Own
                 }
                 (OwnSecretKey, sk, None) => {
@@ -297,7 +372,7 @@ impl Rosenpass {
                         already_set.insert(OwnSecretKey),
                         "secret-key was already set"
                     );
-                    config.secret_key = sk.into();
+                    config.keypair.as_mut().unwrap().secret_key = sk.into();
                     Own
                 }
                 (OwnListen, l, None) => {
@@ -432,10 +507,12 @@ impl Rosenpass {
         };
 
         Self {
-            public_key: "/path/to/rp-public-key".into(),
-            secret_key: "/path/to/rp-secret-key".into(),
+            keypair: Some(Keypair {
+                public_key: "/path/to/rp-public-key".into(),
+                secret_key: "/path/to/rp-secret-key".into(),
+            }),
             peers: vec![peer],
-            ..Self::new("", "")
+            ..Self::new(None)
         }
     }
 }
@@ -448,13 +525,119 @@ impl Default for Verbosity {
 
 #[cfg(test)]
 mod test {
+
     use super::*;
-    use std::net::IpAddr;
+    use std::{borrow::Borrow, net::IpAddr};
+
+    fn toml_des<S: Borrow<str>>(s: S) -> Result<toml::Table, toml::de::Error> {
+        toml::from_str(s.borrow())
+    }
+
+    fn toml_ser<S: Serialize>(s: S) -> Result<toml::Table, toml::ser::Error> {
+        toml::Table::try_from(s)
+    }
+
+    fn assert_toml<L: Serialize, R: Borrow<str>>(l: L, r: R, info: &str) -> anyhow::Result<()> {
+        fn lines_prepend(prefix: &str, s: &str) -> anyhow::Result<String> {
+            use std::fmt::Write;
+
+            let mut buf = String::new();
+            for line in s.lines() {
+                writeln!(&mut buf, "{prefix}{line}")?;
+            }
+            Ok(buf)
+        }
+
+        let l = toml_ser(l)?;
+        let r = toml_des(r.borrow())?;
+        ensure!(
+            l == r,
+            "{}{}TOML value mismatch.\n  Have:\n{}\n  Expected:\n{}",
+            info,
+            if info.is_empty() { "" } else { ": " },
+            lines_prepend("    ", &toml::to_string_pretty(&l)?)?,
+            lines_prepend("    ", &toml::to_string_pretty(&r)?)?
+        );
+        Ok(())
+    }
+
+    fn assert_toml_round<'de, L: Serialize + Deserialize<'de>, R: Borrow<str>>(
+        l: L,
+        r: R,
+    ) -> anyhow::Result<()> {
+        let l = toml_ser(l)?;
+        assert_toml(&l, r.borrow(), "Straight deserialization")?;
+
+        let l: L = l.try_into().unwrap();
+        let l = toml_ser(l).unwrap();
+        assert_toml(l, r.borrow(), "Roundtrip deserialization")?;
+
+        Ok(())
+    }
 
     fn split_str(s: &str) -> Vec<String> {
         s.split(' ').map(|s| s.to_string()).collect()
     }
 
+    #[test]
+    fn toml_serialization() -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        assert_toml_round(
+            Rosenpass::empty(),
+            r#"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+
+            [api]
+            listen_path = []
+            listen_fd = []
+            stream_fd = []
+        "#,
+        )?;
+
+        #[cfg(not(feature = "experiment_api"))]
+        assert_toml_round(
+            Rosenpass::empty(),
+            r#"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+        "#,
+        )?;
+
+        #[cfg(feature = "experiment_api")]
+        assert_toml_round(
+            Rosenpass::from_sk_pk("/my/sk", "/my/pk"),
+            r#"
+            public_key = "/my/pk"
+            secret_key = "/my/sk"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+
+            [api]
+            listen_path = []
+            listen_fd = []
+            stream_fd = []
+        "#,
+        )?;
+
+        #[cfg(not(feature = "experiment_api"))]
+        assert_toml_round(
+            Rosenpass::from_sk_pk("/my/sk", "/my/pk"),
+            r#"
+            public_key = "/my/pk"
+            secret_key = "/my/sk"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+        "#,
+        )?;
+
+        Ok(())
+    }
+
     #[test]
     fn test_simple_cli_parse() {
         let args = split_str(
@@ -465,8 +648,10 @@ mod test {
 
         let config = Rosenpass::parse_args(args).unwrap();
 
-        assert_eq!(config.public_key, PathBuf::from("/my/public-key"));
-        assert_eq!(config.secret_key, PathBuf::from("/my/secret-key"));
+        assert_eq!(
+            config.keypair,
+            Some(Keypair::new("/my/public-key", "/my/secret-key"))
+        );
         assert_eq!(config.verbosity, Verbosity::Verbose);
         assert_eq!(
             &config.listen,
@@ -495,8 +680,10 @@ mod test {
 
         let config = Rosenpass::parse_args(args).unwrap();
 
-        assert_eq!(config.public_key, PathBuf::from("/my/public-key"));
-        assert_eq!(config.secret_key, PathBuf::from("/my/secret-key"));
+        assert_eq!(
+            config.keypair,
+            Some(Keypair::new("/my/public-key", "/my/secret-key"))
+        );
         assert_eq!(config.verbosity, Verbosity::Verbose);
         assert!(&config.listen.is_empty());
         assert_eq!(

rosenpass/src/lib.rs

@@ -1,3 +1,5 @@
+#[cfg(feature = "experiment_api")]
+pub mod api;
 pub mod app_server;
 pub mod cli;
 pub mod config;
@@ -11,4 +13,8 @@ pub enum RosenpassError {
     BufferSizeMismatch,
     #[error("invalid message type")]
     InvalidMessageType(u8),
+    #[error("invalid API message type")]
+    InvalidApiMessageType(u128),
+    #[error("could not parse API message")]
+    InvalidApiMessage,
 }

rosenpass/src/main.rs

@@ -8,6 +8,14 @@ pub fn main() {
     // parse CLI arguments
     let args = CliArgs::parse();
 
+    {
+        use rosenpass_secret_memory as SM;
+        #[cfg(feature = "experiment_memfd_secret")]
+        SM::secret_policy_try_use_memfd_secrets();
+        #[cfg(not(feature = "experiment_memfd_secret"))]
+        SM::secret_policy_use_only_malloc_secrets();
+    }
+
     // init logging
     {
         let mut log_builder = env_logger::Builder::from_default_env(); // sets log level filter from environment (or defaults)
@@ -26,10 +34,11 @@ pub fn main() {
         // error!("error dummy");
     }
 
-    match args.command.run(None) {
+    let broker_interface = args.get_broker_interface();
+    match args.run(broker_interface, None) {
         Ok(_) => {}
         Err(e) => {
-            error!("{e}");
+            error!("{e:?}");
             exit(1);
         }
     }

rosenpass/src/protocol/build_crypto_server.rs

@@ -0,0 +1,127 @@
+use rosenpass_util::{
+    build::Build,
+    mem::{DiscardResultExt, SwapWithDefaultExt},
+    result::ensure_or,
+};
+use thiserror::Error;
+
+use super::{CryptoServer, PeerPtr, SPk, SSk, SymKey};
+
+#[derive(Debug, Clone)]
+pub struct Keypair {
+    pub sk: SSk,
+    pub pk: SPk,
+}
+
+// TODO: We need a named tuple derive
+impl Keypair {
+    pub fn new(sk: SSk, pk: SPk) -> Self {
+        Self { sk, pk }
+    }
+
+    pub fn zero() -> Self {
+        Self::new(SSk::zero(), SPk::zero())
+    }
+
+    pub fn random() -> Self {
+        Self::new(SSk::random(), SPk::random())
+    }
+
+    pub fn from_parts(parts: (SSk, SPk)) -> Self {
+        Self::new(parts.0, parts.1)
+    }
+
+    pub fn into_parts(self) -> (SSk, SPk) {
+        (self.sk, self.pk)
+    }
+}
+
+#[derive(Error, Debug)]
+#[error("PSK already set in BuildCryptoServer")]
+pub struct PskAlreadySet;
+
+#[derive(Error, Debug)]
+#[error("Keypair already set in BuildCryptoServer")]
+pub struct KeypairAlreadySet;
+
+#[derive(Error, Debug)]
+#[error("Can not construct CryptoServer: Missing keypair")]
+pub struct MissingKeypair;
+
+#[derive(Debug, Default)]
+pub struct BuildCryptoServer {
+    pub keypair: Option<Keypair>,
+    pub peers: Vec<PeerParams>,
+}
+
+impl Build<CryptoServer> for BuildCryptoServer {
+    type Error = anyhow::Error;
+
+    fn build(self) -> Result<CryptoServer, Self::Error> {
+        let Some(Keypair { sk, pk }) = self.keypair else {
+            return Err(MissingKeypair)?;
+        };
+
+        let mut srv = CryptoServer::new(sk, pk);
+
+        for (idx, PeerParams { psk, pk }) in self.peers.into_iter().enumerate() {
+            let PeerPtr(idx2) = srv.add_peer(psk, pk)?;
+            assert!(idx == idx2, "Peer id changed during CryptoServer construction from {idx} to {idx2}. This is a developer error.")
+        }
+
+        Ok(srv)
+    }
+}
+
+#[derive(Debug)]
+pub struct PeerParams {
+    pub psk: Option<SymKey>,
+    pub pk: SPk,
+}
+
+impl BuildCryptoServer {
+    pub fn new(keypair: Option<Keypair>, peers: Vec<PeerParams>) -> Self {
+        Self { keypair, peers }
+    }
+
+    pub fn empty() -> Self {
+        Self::new(None, Vec::new())
+    }
+
+    pub fn from_parts(parts: (Option<Keypair>, Vec<PeerParams>)) -> Self {
+        Self {
+            keypair: parts.0,
+            peers: parts.1,
+        }
+    }
+
+    pub fn take_parts(&mut self) -> (Option<Keypair>, Vec<PeerParams>) {
+        (self.keypair.take(), self.peers.swap_with_default())
+    }
+
+    pub fn into_parts(mut self) -> (Option<Keypair>, Vec<PeerParams>) {
+        self.take_parts()
+    }
+
+    pub fn with_keypair(&mut self, keypair: Keypair) -> Result<&mut Self, KeypairAlreadySet> {
+        ensure_or(self.keypair.is_none(), KeypairAlreadySet)?;
+        self.keypair.insert(keypair).discard_result();
+        Ok(self)
+    }
+
+    pub fn with_added_peer(&mut self, psk: Option<SymKey>, pk: SPk) -> &mut Self {
+        // TODO: Check here already whether peer was already added
+        self.peers.push(PeerParams { psk, pk });
+        self
+    }
+
+    pub fn add_peer(&mut self, psk: Option<SymKey>, pk: SPk) -> PeerPtr {
+        let id = PeerPtr(self.peers.len());
+        self.with_added_peer(psk, pk);
+        id
+    }
+
+    pub fn emancipate(&mut self) -> Self {
+        Self::from_parts(self.take_parts())
+    }
+}

rosenpass/src/protocol/mod.rs

@@ -0,0 +1,6 @@
+mod build_crypto_server;
+#[allow(clippy::module_inception)]
+mod protocol;
+
+pub use build_crypto_server::*;
+pub use protocol::*;

rosenpass/src/protocol/protocol.rs

@@ -19,6 +19,7 @@
 //! [CryptoServer].
 //!
 //! ```
+//! use std::ops::DerefMut;
 //! use rosenpass_secret_memory::policy::*;
 //! use rosenpass_cipher_traits::Kem;
 //! use rosenpass_ciphers::kem::StaticKem;
@@ -32,11 +33,11 @@
 //!
 //! // initialize secret and public key for peer a ...
 //! let (mut peer_a_sk, mut peer_a_pk) = (SSk::zero(), SPk::zero());
-//! StaticKem::keygen(peer_a_sk.secret_mut(), peer_a_pk.secret_mut())?;
+//! StaticKem::keygen(peer_a_sk.secret_mut(), peer_a_pk.deref_mut())?;
 //!
 //! // ... and for peer b
 //! let (mut peer_b_sk, mut peer_b_pk) = (SSk::zero(), SPk::zero());
-//! StaticKem::keygen(peer_b_sk.secret_mut(), peer_b_pk.secret_mut())?;
+//! StaticKem::keygen(peer_b_sk.secret_mut(), peer_b_pk.deref_mut())?;
 //!
 //! // initialize server and a pre-shared key
 //! let psk = SymKey::random();
@@ -71,6 +72,7 @@
 
 use std::convert::Infallible;
 use std::mem::size_of;
+use std::ops::Deref;
 use std::{
     collections::hash_map::{
         Entry::{Occupied, Vacant},
@@ -88,20 +90,14 @@ use rosenpass_ciphers::hash_domain::{SecretHashDomain, SecretHashDomainNamespace
 use rosenpass_ciphers::kem::{EphemeralKem, StaticKem};
 use rosenpass_ciphers::{aead, xaead, KEY_LEN};
 use rosenpass_constant_time as constant_time;
-use rosenpass_secret_memory::{Public, Secret};
-use rosenpass_util::{cat, mem::cpy_min, ord::max_usize, time::Timebase};
+use rosenpass_secret_memory::{Public, PublicBox, Secret};
+use rosenpass_util::{cat, mem::cpy_min, time::Timebase};
 use zerocopy::{AsBytes, FromBytes, Ref};
 
 use crate::{hash_domains, msgs::*, RosenpassError};
 
 // CONSTANTS & SETTINGS //////////////////////////
 
-/// Size required to fit any message in binary form
-pub const RTX_BUFFER_SIZE: usize = max_usize(
-    size_of::<Envelope<InitHello>>(),
-    size_of::<Envelope<InitConf>>(),
-);
-
 /// A type for time, e.g. for backoff before re-tries
 pub type Timing = f64;
 
@@ -138,11 +134,10 @@ pub const PEER_COOKIE_VALUE_EPOCH: Timing = 120.0;
 // decryption for a second epoch
 pub const BISCUIT_EPOCH: Timing = 300.0;
 
-// Retransmission pub constants; will retransmit for up to _ABORT ms; starting with a delay of
-// _DELAY_BEG ms and increasing the delay exponentially by a factor of
-// _DELAY_GROWTH up to _DELAY_END. An additional jitter factor of ±_DELAY_JITTER
-// is added.
-pub const RETRANSMIT_ABORT: Timing = 120.0;
+// Retransmission pub constants; will retransmit for up to _ABORT seconds;
+// starting with a delay of _DELAY_BEGIN seconds and increasing the delay
+// exponentially by a factor of _DELAY_GROWTH up to _DELAY_END.
+// An additional jitter factor of ±_DELAY_JITTER is added.
 pub const RETRANSMIT_DELAY_GROWTH: Timing = 2.0;
 pub const RETRANSMIT_DELAY_BEGIN: Timing = 0.5;
 pub const RETRANSMIT_DELAY_END: Timing = 10.0;
@@ -163,7 +158,7 @@ pub fn has_happened(ev: Timing, now: Timing) -> bool {
 
 // DATA STRUCTURES & BASIC TRAITS & ACCESSORS ////
 
-pub type SPk = Secret<{ StaticKem::PK_LEN }>; // Just Secret<> instead of Public<> so it gets allocated on the heap
+pub type SPk = PublicBox<{ StaticKem::PK_LEN }>;
 pub type SSk = Secret<{ StaticKem::SK_LEN }>;
 pub type EPk = Public<{ EphemeralKem::PK_LEN }>;
 pub type ESk = Secret<{ EphemeralKem::SK_LEN }>;
@@ -548,7 +543,7 @@ impl CryptoServer {
     pub fn pidm(&self) -> Result<PeerId> {
         Ok(Public::new(
             hash_domains::peerid()?
-                .mix(self.spkm.secret())?
+                .mix(self.spkm.deref())?
                 .into_value()))
     }
 
@@ -708,7 +703,7 @@ impl Peer {
     pub fn pidt(&self) -> Result<PeerId> {
         Ok(Public::new(
             hash_domains::peerid()?
-                .mix(self.spkt.secret())?
+                .mix(self.spkt.deref())?
                 .into_value()))
     }
 }
@@ -1017,7 +1012,7 @@ impl CryptoServer {
 
         let cookie_value = active_cookie_value.unwrap();
         let cookie_key = hash_domains::cookie_key()?
-            .mix(self.spkm.secret())?
+            .mix(self.spkm.deref())?
             .into_value();
 
         let mut msg_out = truncating_cast_into::<CookieReply>(tx_buf)?;
@@ -1477,7 +1472,7 @@ impl IniHsPtr {
                         .min(ih.tx_count as f64),
                 )
                 * RETRANSMIT_DELAY_JITTER
-                * (rand::random::<f64>() + 1.0); // TODO: Replace with the rand crate
+                * (rand::random::<f64>() + 1.0);
         ih.tx_count += 1;
         Ok(())
     }
@@ -1509,7 +1504,7 @@ where
     /// Calculate the message authentication code (`mac`) and also append cookie value
     pub fn seal(&mut self, peer: PeerPtr, srv: &CryptoServer) -> Result<()> {
         let mac = hash_domains::mac()?
-            .mix(peer.get(srv).spkt.secret())?
+            .mix(peer.get(srv).spkt.deref())?
             .mix(&self.as_bytes()[span_of!(Self, msg_type..mac)])?;
         self.mac.copy_from_slice(mac.into_value()[..16].as_ref());
         self.seal_cookie(peer, srv)?;
@@ -1536,7 +1531,7 @@ where
     /// Check the message authentication code
     pub fn check_seal(&self, srv: &CryptoServer) -> Result<bool> {
         let expected = hash_domains::mac()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(&self.as_bytes()[span_of!(Self, msg_type..mac)])?;
         Ok(constant_time::memcmp(
             &self.mac,
@@ -1641,7 +1636,7 @@ impl HandshakeState {
 
         // calculate ad contents
         let ad = hash_domains::biscuit_ad()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(self.sidi.as_slice())?
             .mix(self.sidr.as_slice())?
             .into_value();
@@ -1676,7 +1671,7 @@ impl HandshakeState {
 
         // Calculate additional data fields
         let ad = hash_domains::biscuit_ad()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(sidi.as_slice())?
             .mix(sidr.as_slice())?
             .into_value();
@@ -1763,7 +1758,7 @@ impl CryptoServer {
         let mut hs = InitiatorHandshake::zero_with_timestamp(self);
 
         // IHI1
-        hs.core.init(peer.get(self).spkt.secret())?;
+        hs.core.init(peer.get(self).spkt.deref())?;
 
         // IHI2
         hs.core.sidi.randomize();
@@ -1780,7 +1775,7 @@ impl CryptoServer {
         hs.core
             .encaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
                 ih.sctr.as_mut_slice(),
-                peer.get(self).spkt.secret(),
+                peer.get(self).spkt.deref(),
             )?;
 
         // IHI6
@@ -1789,7 +1784,7 @@ impl CryptoServer {
 
         // IHI7
         hs.core
-            .mix(self.spkm.secret())?
+            .mix(self.spkm.deref())?
             .mix(peer.get(self).psk.secret())?;
 
         // IHI8
@@ -1807,7 +1802,7 @@ impl CryptoServer {
         core.sidi = SessionId::from_slice(&ih.sidi);
 
         // IHR1
-        core.init(self.spkm.secret())?;
+        core.init(self.spkm.deref())?;
 
         // IHR4
         core.mix(&ih.sidi)?.mix(&ih.epki)?;
@@ -1815,7 +1810,7 @@ impl CryptoServer {
         // IHR5
         core.decaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             self.sskm.secret(),
-            self.spkm.secret(),
+            self.spkm.deref(),
             &ih.sctr,
         )?;
 
@@ -1828,7 +1823,7 @@ impl CryptoServer {
         };
 
         // IHR7
-        core.mix(peer.get(self).spkt.secret())?
+        core.mix(peer.get(self).spkt.deref())?
             .mix(peer.get(self).psk.secret())?;
 
         // IHR8
@@ -1848,7 +1843,7 @@ impl CryptoServer {
         // RHR5
         core.encaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             &mut rh.scti,
-            peer.get(self).spkt.secret(),
+            peer.get(self).spkt.deref(),
         )?;
 
         // RHR6
@@ -1909,14 +1904,14 @@ impl CryptoServer {
         // RHI4
         core.decaps_and_mix::<EphemeralKem, { EphemeralKem::SHK_LEN }>(
             hs!().eski.secret(),
-            &*hs!().epki,
+            hs!().epki.deref(),
             &rh.ecti,
         )?;
 
         // RHI5
         core.decaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             self.sskm.secret(),
-            self.spkm.secret(),
+            self.spkm.deref(),
             &rh.scti,
         )?;
 
@@ -2014,8 +2009,7 @@ impl CryptoServer {
 
         // Send ack – Implementing sending the empty acknowledgement here
         // instead of a generic PeerPtr::send(&Server, Option<&[u8]>) -> Either<EmptyData, Data>
-        // because data transmission is a stub currently. This software is supposed to be used
-        // as a key exchange service feeding a PSK into some classical (i.e. non post quantum)
+        // because data transmission is a stub currently.
         let ses = peer
             .session()
             .get_mut(self)
@@ -2113,7 +2107,7 @@ impl CryptoServer {
                     ),
                 }?;
 
-                let spkt = peer.get(self).spkt.secret();
+                let spkt = peer.get(self).spkt.deref();
                 let cookie_key = hash_domains::cookie_key()?.mix(spkt)?.into_value();
                 let cookie_value = peer.cv().update_mut(self).unwrap();
 
@@ -2146,7 +2140,7 @@ fn truncating_cast_into_nomut<T: FromBytes>(buf: &[u8]) -> Result<Ref<&[u8], T>,
 
 #[cfg(test)]
 mod test {
-    use std::{net::SocketAddrV4, thread::sleep, time::Duration};
+    use std::{net::SocketAddrV4, ops::DerefMut, thread::sleep, time::Duration};
 
     use super::*;
     use serial_test::serial;
@@ -2255,7 +2249,7 @@ mod test {
     fn keygen() -> Result<(SSk, SPk)> {
         // TODO: Copied from the benchmark; deduplicate
         let (mut sk, mut pk) = (SSk::zero(), SPk::zero());
-        StaticKem::keygen(sk.secret_mut(), pk.secret_mut())?;
+        StaticKem::keygen(sk.secret_mut(), pk.deref_mut())?;
         Ok((sk, pk))
     }
 

rosenpass/tests/api-integration-tests-api-setup.rs

@@ -0,0 +1,332 @@
+use std::{
+    borrow::Borrow,
+    io::{BufRead, BufReader, Write},
+    os::unix::net::UnixStream,
+    process::Stdio,
+    thread::sleep,
+    time::Duration,
+};
+
+use anyhow::{bail, Context};
+use command_fds::{CommandFdExt, FdMapping};
+use hex_literal::hex;
+use rosenpass::api::{
+    self, add_listen_socket_response_status, add_psk_broker_response_status,
+    supply_keypair_response_status,
+};
+use rosenpass_util::{
+    b64::B64Display,
+    file::LoadValueB64,
+    io::IoErrorKind,
+    length_prefix_encoding::{decoder::LengthPrefixDecoder, encoder::LengthPrefixEncoder},
+    mem::{DiscardResultExt, MoveExt},
+    mio::WriteWithFileDescriptors,
+    zerocopy::ZerocopySliceExt,
+};
+use rustix::fd::{AsFd, AsRawFd};
+use tempfile::TempDir;
+use zerocopy::AsBytes;
+
+use rosenpass::protocol::SymKey;
+
+struct KillChild(std::process::Child);
+
+impl Drop for KillChild {
+    fn drop(&mut self) {
+        self.0.kill().discard_result();
+        self.0.wait().discard_result()
+    }
+}
+
+#[test]
+fn api_integration_api_setup() -> anyhow::Result<()> {
+    rosenpass_secret_memory::policy::secret_policy_use_only_malloc_secrets();
+
+    let dir = TempDir::with_prefix("rosenpass-api-integration-test")?;
+
+    macro_rules! tempfile {
+        ($($lst:expr),+) => {{
+            let mut buf =  dir.path().to_path_buf();
+            $(buf.push($lst);)*
+            buf
+        }}
+    }
+
+    let peer_a_endpoint = "[::1]:0";
+    let peer_a_listen = std::net::UdpSocket::bind(peer_a_endpoint)?;
+    let peer_a_endpoint = format!("{}", peer_a_listen.local_addr()?);
+    let peer_a_keypair = config::Keypair::new(tempfile!("a.pk"), tempfile!("a.sk"));
+
+    let peer_b_osk = tempfile!("b.osk");
+    let peer_b_wg_device = "mock_device";
+    let peer_b_wg_peer_id = hex!(
+        "
+        93 0f ee 77 0c 6b 54 7e  13 5f 13 92 21 97 26 53
+        7d 77 4a 6a 0f 6c eb 1a  dd 6e 5b c4 1b 92 cd 99
+    "
+    );
+
+    use rosenpass::config;
+    let peer_a = config::Rosenpass {
+        config_file_path: tempfile!("a.config"),
+        keypair: None,
+        listen: vec![], // TODO: This could collide by accident
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("a.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("b.pk"),
+            key_out: None,
+            endpoint: None,
+            pre_shared_key: None,
+            wg: Some(config::WireGuard {
+                device: peer_b_wg_device.to_string(),
+                peer: format!("{}", peer_b_wg_peer_id.fmt_b64::<8129>()),
+                extra_params: vec![],
+            }),
+        }],
+    };
+
+    let peer_b_keypair = config::Keypair::new(tempfile!("b.pk"), tempfile!("b.sk"));
+    let peer_b = config::Rosenpass {
+        config_file_path: tempfile!("b.config"),
+        keypair: Some(peer_b_keypair.clone()),
+        listen: vec![],
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("b.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("a.pk"),
+            key_out: Some(peer_b_osk.clone()),
+            endpoint: Some(peer_a_endpoint.to_owned()),
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    // Generate the keys
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_a_keypair.secret_key.clone(),
+        peer_a_keypair.public_key.clone(),
+    )?;
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_b_keypair.secret_key.clone(),
+        peer_b_keypair.public_key.clone(),
+    )?;
+
+    // Write the configuration files
+    peer_a.commit()?;
+    peer_b.commit()?;
+
+    let (deliberate_fail_api_client, deliberate_fail_api_server) =
+        std::os::unix::net::UnixStream::pair()?;
+    let deliberate_fail_child_fd = 3;
+
+    // Start peer a
+    let _proc_a = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args(["--api-stream-fd", &deliberate_fail_child_fd.to_string()])
+            .fd_mappings(vec![FdMapping {
+                parent_fd: deliberate_fail_api_server.move_here().as_raw_fd(),
+                child_fd: 3,
+            }])?
+            .args([
+                "exchange-config",
+                peer_a.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?,
+    );
+
+    // Start peer b
+    let mut proc_b = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_b.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stderr(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Acquire stdout
+    let mut out_b = BufReader::new(proc_b.0.stdout.take().context("")?).lines();
+
+    // Now connect to the peers
+    let api_path = peer_a.api.listen_path[0].as_path();
+
+    // Wait for the socket to be created
+    let attempt = 0;
+    while !api_path.exists() {
+        sleep(Duration::from_millis(200));
+        assert!(
+            attempt < 50,
+            "Api failed to be created even after 50 seconds"
+        );
+    }
+
+    let api = UnixStream::connect(api_path)?;
+    let (psk_broker_sock, psk_broker_server_sock) = UnixStream::pair()?;
+
+    // Send AddListenSocket request
+    {
+        let fd = peer_a_listen.as_fd();
+
+        let mut fds = vec![&fd].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::AddListenSocketRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+        std::mem::forget(peer_a_listen);
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::AddListenSocketResponse>()?;
+        assert_eq!(
+            *res,
+            api::AddListenSocketResponse::new(add_listen_socket_response_status::OK)
+        );
+    }
+
+    // Deliberately break API connection given via FD; this checks that the
+    // API connections are closed when invalid data is received and it also
+    // implicitly checks that other connections are unaffected
+    {
+        use std::io::ErrorKind as K;
+        let client = deliberate_fail_api_client;
+        let err = loop {
+            if let Err(e) = client.borrow().write(&[0xffu8; 16]) {
+                break e;
+            }
+        };
+        // NotConnected happens on Mac
+        assert!(matches!(
+            err.io_error_kind(),
+            K::ConnectionReset | K::BrokenPipe | K::NotConnected
+        ));
+    }
+
+    // Send SupplyKeypairRequest
+    {
+        use rustix::fs::{open, Mode, OFlags};
+        let sk = open(peer_a_keypair.secret_key, OFlags::RDONLY, Mode::empty())?;
+        let pk = open(peer_a_keypair.public_key, OFlags::RDONLY, Mode::empty())?;
+
+        let mut fds = vec![&sk, &pk].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::SupplyKeypairRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::SupplyKeypairResponse>()?;
+        assert_eq!(
+            *res,
+            api::SupplyKeypairResponse::new(supply_keypair_response_status::OK)
+        );
+    }
+
+    // Send AddPskBroker request
+    {
+        let mut fds = vec![psk_broker_server_sock.as_fd()].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::AddPskBrokerRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::AddPskBrokerResponse>()?;
+        assert_eq!(
+            *res,
+            api::AddPskBrokerResponse::new(add_psk_broker_response_status::OK)
+        );
+    }
+
+    // Wait for the keys to successfully exchange a key
+    let mut attempt = 0;
+    loop {
+        // Read OSK generated by A
+        let osk_a = {
+            use rosenpass_wireguard_broker::api::msgs as M;
+            type SetPskReqPkg = M::Envelope<M::SetPskRequest>;
+            type SetPskResPkg = M::Envelope<M::SetPskResponse>;
+
+            // Receive request
+            let mut decoder = LengthPrefixDecoder::new([0u8; M::REQUEST_MSG_BUFFER_SIZE]);
+            let req = decoder.read_all_from_stdio(&psk_broker_sock)?;
+
+            let req = req.zk_parse::<SetPskReqPkg>()?;
+            assert_eq!(req.msg_type, M::MsgType::SetPsk as u8);
+            assert_eq!(req.payload.peer_id, peer_b_wg_peer_id);
+            assert_eq!(req.payload.iface()?, peer_b_wg_device);
+
+            // Send response
+            let res = SetPskResPkg {
+                msg_type: M::MsgType::SetPsk as u8,
+                reserved: [0u8; 3],
+                payload: M::SetPskResponse {
+                    return_code: M::SetPskResponseReturnCode::Success as u8,
+                },
+            };
+            LengthPrefixEncoder::from_message(res.as_bytes())
+                .write_all_to_stdio(&psk_broker_sock)?;
+
+            SymKey::from_slice(&req.payload.psk)
+        };
+
+        // Read OSK generated by B
+        let osk_b = {
+            let line = out_b.next().context("")??;
+            let words = line.split(' ').collect::<Vec<_>>();
+
+            // FIXED     FIXED PEER-ID                                      FIXED    FILENAME       STATUS
+            // output-key peer KZqXTZ4l2aNnkJtLPhs4D8JxHTGmRSL9w3Qr+X8JxFk= key-file "client-A-osk" exchanged
+            let peer_id = words
+                .get(2)
+                .with_context(|| format!("Bad rosenpass output: `{line}`"))?;
+            assert_eq!(
+                line,
+                format!(
+                    "output-key peer {peer_id} key-file \"{}\" exchanged",
+                    peer_b_osk.to_str().context("")?
+                )
+            );
+
+            SymKey::load_b64::<64, _>(peer_b_osk.clone())?
+        };
+
+        // TODO: This may be flaky. Both rosenpass instances are not guaranteed to produce
+        // the same number of output events; they merely guarantee eventual consistency of OSK.
+        // Correctly, we should use tokio to read any number of generated OSKs and indicate
+        // success on consensus
+        match osk_a.secret() == osk_b.secret() {
+            true => break,
+            false if attempt > 10 => bail!("Peers did not produce a matching key even after ten attempts. Something is wrong with the key exchange!"),
+            false => {},
+        };
+
+        attempt += 1;
+    }
+
+    Ok(())
+}

rosenpass/tests/api-integration-tests.rs

@@ -0,0 +1,194 @@
+use std::{
+    io::{BufRead, BufReader},
+    net::ToSocketAddrs,
+    os::unix::net::UnixStream,
+    process::Stdio,
+};
+
+use anyhow::{bail, Context};
+use rosenpass::api;
+use rosenpass_to::{ops::copy_slice_least_src, To};
+use rosenpass_util::{
+    file::LoadValueB64,
+    length_prefix_encoding::{decoder::LengthPrefixDecoder, encoder::LengthPrefixEncoder},
+};
+use rosenpass_util::{mem::DiscardResultExt, zerocopy::ZerocopySliceExt};
+use tempfile::TempDir;
+use zerocopy::AsBytes;
+
+use rosenpass::protocol::SymKey;
+
+struct KillChild(std::process::Child);
+
+impl Drop for KillChild {
+    fn drop(&mut self) {
+        self.0.kill().discard_result();
+        self.0.wait().discard_result()
+    }
+}
+
+#[test]
+fn api_integration_test() -> anyhow::Result<()> {
+    rosenpass_secret_memory::policy::secret_policy_use_only_malloc_secrets();
+
+    let dir = TempDir::with_prefix("rosenpass-api-integration-test")?;
+
+    macro_rules! tempfile {
+        ($($lst:expr),+) => {{
+            let mut buf =  dir.path().to_path_buf();
+            $(buf.push($lst);)*
+            buf
+        }}
+    }
+
+    let peer_a_endpoint = "[::1]:61423";
+    let peer_a_osk = tempfile!("a.osk");
+    let peer_b_osk = tempfile!("b.osk");
+
+    use rosenpass::config;
+
+    let peer_a_keypair = config::Keypair::new(tempfile!("a.pk"), tempfile!("a.sk"));
+    let peer_a = config::Rosenpass {
+        config_file_path: tempfile!("a.config"),
+        keypair: Some(peer_a_keypair.clone()),
+        listen: peer_a_endpoint.to_socket_addrs()?.collect(), // TODO: This could collide by accident
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("a.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("b.pk"),
+            key_out: Some(peer_a_osk.clone()),
+            endpoint: None,
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    let peer_b_keypair = config::Keypair::new(tempfile!("b.pk"), tempfile!("b.sk"));
+    let peer_b = config::Rosenpass {
+        config_file_path: tempfile!("b.config"),
+        keypair: Some(peer_b_keypair.clone()),
+        listen: vec![],
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("b.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("a.pk"),
+            key_out: Some(peer_b_osk.clone()),
+            endpoint: Some(peer_a_endpoint.to_owned()),
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    // Generate the keys
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_a_keypair.secret_key.clone(),
+        peer_a_keypair.public_key.clone(),
+    )?;
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_b_keypair.secret_key.clone(),
+        peer_b_keypair.public_key.clone(),
+    )?;
+
+    // Write the configuration files
+    peer_a.commit()?;
+    peer_b.commit()?;
+
+    // Start peer a
+    let mut proc_a = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_a.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Start peer b
+    let mut proc_b = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_b.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Acquire stdout
+    let mut out_a = BufReader::new(proc_a.0.stdout.take().context("")?).lines();
+    let mut out_b = BufReader::new(proc_b.0.stdout.take().context("")?).lines();
+
+    // Wait for the keys to successfully exchange a key
+    let mut attempt = 0;
+    loop {
+        let line_a = out_a.next().context("")??;
+        let line_b = out_b.next().context("")??;
+
+        let words_a = line_a.split(' ').collect::<Vec<_>>();
+        let words_b = line_b.split(' ').collect::<Vec<_>>();
+
+        // FIXED     FIXED PEER-ID                                      FIXED    FILENAME       STATUS
+        // output-key peer KZqXTZ4l2aNnkJtLPhs4D8JxHTGmRSL9w3Qr+X8JxFk= key-file "client-A-osk" exchanged
+        let peer_a_id = words_b
+            .get(2)
+            .with_context(|| format!("Bad rosenpass output: `{line_b}`"))?;
+        let peer_b_id = words_a
+            .get(2)
+            .with_context(|| format!("Bad rosenpass output: `{line_a}`"))?;
+        assert_eq!(
+            line_a,
+            format!(
+                "output-key peer {peer_b_id} key-file \"{}\" exchanged",
+                peer_a_osk.to_str().context("")?
+            )
+        );
+        assert_eq!(
+            line_b,
+            format!(
+                "output-key peer {peer_a_id} key-file \"{}\" exchanged",
+                peer_b_osk.to_str().context("")?
+            )
+        );
+
+        // Read OSKs
+        let osk_a = SymKey::load_b64::<64, _>(peer_a_osk.clone())?;
+        let osk_b = SymKey::load_b64::<64, _>(peer_b_osk.clone())?;
+        match osk_a.secret() == osk_b.secret() {
+            true => break,
+            false if attempt > 10 => bail!("Peers did not produce a matching key even after ten attempts. Something is wrong with the key exchange!"),
+            false => {},
+        };
+
+        attempt += 1;
+    }
+
+    // Now connect to the peers
+    let api_a = UnixStream::connect(&peer_a.api.listen_path[0])?;
+    let api_b = UnixStream::connect(&peer_b.api.listen_path[0])?;
+
+    for conn in ([api_a, api_b]).iter() {
+        let mut echo = [0u8; 256];
+        copy_slice_least_src("Hello World".as_bytes()).to(&mut echo);
+
+        let req = api::PingRequest::new(echo);
+        LengthPrefixEncoder::from_message(req.as_bytes()).write_all_to_stdio(conn)?;
+
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(conn)?;
+        let res = res.zk_parse::<api::PingResponse>()?;
+        assert_eq!(*res, api::PingResponse::new(echo));
+    }
+
+    Ok(())
+}

rosenpass/tests/integration_test.rs

@@ -15,9 +15,19 @@ use std::io::Write;
 
 const BIN: &str = "rosenpass";
 
+fn setup_tests() {
+    use rosenpass_secret_memory as SM;
+    #[cfg(feature = "experiment_memfd_secret")]
+    SM::secret_policy_try_use_memfd_secrets();
+    #[cfg(not(feature = "experiment_memfd_secret"))]
+    SM::secret_policy_use_only_malloc_secrets();
+}
+
 // check that we can generate keys
 #[test]
 fn generate_keys() {
+    setup_tests();
+
     let tmpdir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("keygen");
     fs::create_dir_all(&tmpdir).unwrap();
 
@@ -94,14 +104,11 @@ fn run_server_client_exchange(
     .unwrap();
 
     std::thread::spawn(move || {
-        cli.command
-            .run(Some(
-                server_test_builder
-                    .termination_handler(Some(server_terminate_rx))
-                    .build()
-                    .unwrap(),
-            ))
+        let test_helpers = server_test_builder
+            .termination_handler(Some(server_terminate_rx))
+            .build()
             .unwrap();
+        cli.run(None, Some(test_helpers)).unwrap();
     });
 
     let cli = CliArgs::try_parse_from(
@@ -112,14 +119,11 @@ fn run_server_client_exchange(
     .unwrap();
 
     std::thread::spawn(move || {
-        cli.command
-            .run(Some(
-                client_test_builder
-                    .termination_handler(Some(client_terminate_rx))
-                    .build()
-                    .unwrap(),
-            ))
+        let test_helpers = client_test_builder
+            .termination_handler(Some(client_terminate_rx))
+            .build()
             .unwrap();
+        cli.run(None, Some(test_helpers)).unwrap();
     });
 
     // give them some time to do the key exchange under load
@@ -134,6 +138,7 @@ fn run_server_client_exchange(
 #[test]
 #[serial]
 fn check_exchange_under_normal() {
+    setup_tests();
     setup_logging();
 
     let tmpdir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("exchange");
@@ -206,6 +211,7 @@ fn check_exchange_under_normal() {
 #[test]
 #[serial]
 fn check_exchange_under_dos() {
+    setup_tests();
     setup_logging();
 
     //Generate binary with responder with feature integration_test
@@ -283,9 +289,11 @@ struct MockBrokerInner {
     interface: Option<String>,
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Default)]
 struct MockBroker {
     inner: Arc<Mutex<MockBrokerInner>>,
+    mio_token: Option<mio::Token>,
 }
 
 impl WireguardBrokerMio for MockBroker {
@@ -294,8 +302,9 @@ impl WireguardBrokerMio for MockBroker {
     fn register(
         &mut self,
         _registry: &mio::Registry,
-        _token: mio::Token,
+        token: mio::Token,
     ) -> Result<(), Self::MioError> {
+        self.mio_token = Some(token);
         Ok(())
     }
 
@@ -304,8 +313,13 @@ impl WireguardBrokerMio for MockBroker {
     }
 
     fn unregister(&mut self, _registry: &mio::Registry) -> Result<(), Self::MioError> {
+        self.mio_token = None;
         Ok(())
     }
+
+    fn mio_token(&self) -> Option<mio::Token> {
+        self.mio_token
+    }
 }
 
 impl rosenpass_wireguard_broker::WireGuardBroker for MockBroker {
@@ -321,7 +335,7 @@ impl rosenpass_wireguard_broker::WireGuardBroker for MockBroker {
             if let Ok(ref mut mutex) = lock {
                 **mutex = MockBrokerInner {
                     psk: Some(config.psk.clone()),
-                    peer_id: Some(config.peer_id.clone()),
+                    peer_id: Some(*config.peer_id),
                     interface: Some(std::str::from_utf8(config.interface).unwrap().to_string()),
                 };
                 break Ok(());

rp/Cargo.toml

@@ -20,9 +20,9 @@ rosenpass-ciphers = { workspace = true }
 rosenpass-cipher-traits = { workspace = true }
 rosenpass-secret-memory = { workspace = true }
 rosenpass-util = { workspace = true }
-rosenpass-wireguard-broker = {workspace = true}
+rosenpass-wireguard-broker = { workspace = true }
 
-tokio = {workspace = true}
+tokio = { workspace = true }
 
 [target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies]
 ctrlc-async = "3.2"
@@ -35,8 +35,9 @@ netlink-packet-generic = "0.3"
 netlink-packet-wireguard = "0.2"
 
 [dev-dependencies]
-tempfile = {workspace = true}
-stacker = {workspace = true}
+tempfile = { workspace = true }
+stacker = { workspace = true }
 
 [features]
-enable_memfd_alloc = []
+experiment_memfd_secret = []
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]

rp/src/exchange.rs

@@ -2,7 +2,9 @@ use std::{net::SocketAddr, path::PathBuf};
 
 use anyhow::Result;
 
+#[cfg(any(target_os = "linux", target_os = "freebsd"))]
 use crate::key::WG_B64_LEN;
+
 #[derive(Default)]
 pub struct ExchangePeer {
     pub public_keys_dir: PathBuf,
@@ -186,8 +188,7 @@ pub async fn exchange(options: ExchangeOptions) -> Result<()> {
     let pk = SPk::load(&pqpk)?;
 
     let mut srv = Box::new(AppServer::new(
-        sk,
-        pk,
+        Some((sk, pk)),
         if let Some(listen) = options.listen {
             vec![listen]
         } else {

rp/src/key.rs

@@ -1,11 +1,12 @@
 use std::{
     fs::{self, DirBuilder},
+    ops::DerefMut,
     os::unix::fs::{DirBuilderExt, PermissionsExt},
     path::Path,
 };
 
 use anyhow::{anyhow, Result};
-use rosenpass_util::file::{LoadValueB64, StoreValueB64};
+use rosenpass_util::file::{LoadValueB64, StoreValue, StoreValueB64};
 use zeroize::Zeroize;
 
 use rosenpass::protocol::{SPk, SSk};
@@ -56,8 +57,8 @@ pub fn genkey(private_keys_dir: &Path) -> Result<()> {
     if !pqsk_path.exists() && !pqpk_path.exists() {
         let mut pqsk = SSk::random();
         let mut pqpk = SPk::random();
-        StaticKem::keygen(pqsk.secret_mut(), pqpk.secret_mut())?;
-        pqpk.store_secret(pqpk_path)?;
+        StaticKem::keygen(pqsk.secret_mut(), pqpk.deref_mut())?;
+        pqpk.store(pqpk_path)?;
         pqsk.store_secret(pqsk_path)?;
     } else {
         eprintln!(

rp/src/main.rs

@@ -11,9 +11,9 @@ mod key;
 
 #[tokio::main]
 async fn main() {
-    #[cfg(feature = "enable_memfd_alloc")]
+    #[cfg(feature = "experiment_memfd_secret")]
     policy::secret_policy_try_use_memfd_secrets();
-    #[cfg(not(feature = "enable_memfd_alloc"))]
+    #[cfg(not(feature = "experiment_memfd_secret"))]
     policy::secret_policy_use_only_malloc_secrets();
 
     let cli = match Cli::parse(std::env::args().peekable()) {

secret-memory/Cargo.toml

@@ -21,6 +21,6 @@ log = { workspace = true }
 
 [dev-dependencies]
 allocator-api2-tests = { workspace = true }
-tempfile = {workspace = true}
-base64ct = {workspace = true}
-procspawn = {workspace = true}
+tempfile = { workspace = true }
+base64ct = { workspace = true }
+procspawn = { workspace = true }

secret-memory/src/lib.rs

@@ -6,6 +6,7 @@ pub mod alloc;
 
 mod public;
 pub use crate::public::Public;
+pub use crate::public::PublicBox;
 
 mod secret;
 pub use crate::secret::Secret;

secret-memory/src/public.rs

@@ -172,12 +172,154 @@ impl<const N: usize> StoreValueB64Writer for Public<N> {
     }
 }
 
+#[derive(Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[repr(transparent)]
+pub struct PublicBox<const N: usize> {
+    pub inner: Box<Public<N>>,
+}
+
+impl<const N: usize> PublicBox<N> {
+    /// Create a new [PublicBox] from a byte slice
+    pub fn from_slice(value: &[u8]) -> Self {
+        Self {
+            inner: Box::new(Public::from_slice(value)),
+        }
+    }
+
+    /// Create a new [PublicBox] from a byte array
+    pub fn new(value: [u8; N]) -> Self {
+        Self {
+            inner: Box::new(Public::new(value)),
+        }
+    }
+
+    /// Create a zero initialized [PublicBox]
+    pub fn zero() -> Self {
+        Self {
+            inner: Box::new(Public::zero()),
+        }
+    }
+
+    /// Create a random initialized [PublicBox]
+    pub fn random() -> Self {
+        Self {
+            inner: Box::new(Public::random()),
+        }
+    }
+
+    /// Randomize all bytes in an existing [PublicBox]
+    pub fn randomize(&mut self) {
+        self.inner.randomize()
+    }
+}
+
+impl<const N: usize> Randomize for PublicBox<N> {
+    fn try_fill<R: Rng + ?Sized>(&mut self, rng: &mut R) -> Result<(), rand::Error> {
+        self.inner.try_fill(rng)
+    }
+}
+
+impl<const N: usize> fmt::Debug for PublicBox<N> {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        debug_crypto_array(&**self, fmt)
+    }
+}
+
+impl<const N: usize> Deref for PublicBox<N> {
+    type Target = [u8; N];
+
+    fn deref(&self) -> &[u8; N] {
+        self.inner.deref()
+    }
+}
+
+impl<const N: usize> DerefMut for PublicBox<N> {
+    fn deref_mut(&mut self) -> &mut [u8; N] {
+        self.inner.deref_mut()
+    }
+}
+
+impl<const N: usize> Borrow<[u8]> for PublicBox<N> {
+    fn borrow(&self) -> &[u8] {
+        self.deref()
+    }
+}
+
+impl<const N: usize> BorrowMut<[u8]> for PublicBox<N> {
+    fn borrow_mut(&mut self) -> &mut [u8] {
+        self.deref_mut()
+    }
+}
+
+impl<const N: usize> LoadValue for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    // This is implemented separately from Public to avoid allocating too much stack memory
+    fn load<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
+        let mut p = Self::random();
+        fopen_r(path)?.read_exact_to_end(p.deref_mut())?;
+        Ok(p)
+    }
+}
+
+impl<const N: usize> StoreValue for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store<P: AsRef<Path>>(&self, path: P) -> anyhow::Result<()> {
+        self.inner.store(path)
+    }
+}
+
+impl<const N: usize> LoadValueB64 for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    // This is implemented separately from Public to avoid allocating too much stack memory
+    fn load_b64<const F: usize, P: AsRef<Path>>(path: P) -> Result<Self, Self::Error>
+    where
+        Self: Sized,
+    {
+        // A vector is used here to ensure heap allocation without copy from stack
+        let mut f = vec![0u8; F];
+        let mut v = PublicBox::zero();
+        let p = path.as_ref();
+
+        let len = fopen_r(p)?
+            .read_slice_to_end(&mut f)
+            .with_context(|| format!("Could not load file {p:?}"))?;
+
+        b64_decode(&f[0..len], v.deref_mut())
+            .with_context(|| format!("Could not decode base64 file {p:?}"))?;
+
+        Ok(v)
+    }
+}
+
+impl<const N: usize> StoreValueB64 for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store_b64<const F: usize, P: AsRef<Path>>(&self, path: P) -> anyhow::Result<()> {
+        self.inner.store_b64::<F, P>(path)
+    }
+}
+
+impl<const N: usize> StoreValueB64Writer for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store_b64_writer<const F: usize, W: std::io::Write>(
+        &self,
+        writer: W,
+    ) -> Result<(), Self::Error> {
+        self.inner.store_b64_writer::<F, W>(writer)
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
     #[cfg(test)]
+    #[allow(clippy::module_inception)]
     mod tests {
-        use crate::Public;
+        use crate::{Public, PublicBox};
         use rosenpass_util::{
             b64::b64_encode,
             file::{
@@ -185,32 +327,35 @@ mod tests {
                 Visibility,
             },
         };
-        use std::{fs, os::unix::fs::PermissionsExt};
+        use std::{fs, ops::Deref, os::unix::fs::PermissionsExt};
         use tempfile::tempdir;
 
-        /// test loading a public from an example file, and then storing it again in a different file
-        #[test]
-        fn test_public_load_store() {
-            const N: usize = 100;
+        /// Number of bytes in payload for load and store tests
+        const N: usize = 100;
 
+        /// Convenience function for running a load/store test
+        fn run_load_store_test<
+            T: LoadValue<Error = anyhow::Error>
+                + StoreValue<Error = anyhow::Error>
+                + Deref<Target = [u8; N]>,
+        >() {
             // Generate original random bytes
             let original_bytes: [u8; N] = [rand::random(); N];
 
             // Create a temporary directory
             let temp_dir = tempdir().unwrap();
 
-            // Store the original public to an example file in the temporary directory
+            // Store the original bytes to an example file in the temporary directory
             let example_file = temp_dir.path().join("example_file");
-            std::fs::write(example_file.clone(), &original_bytes).unwrap();
+            std::fs::write(&example_file, original_bytes).unwrap();
 
-            // Load the public from the example file
+            // Load the value from the example file into our generic type
+            let loaded_public = T::load(&example_file).unwrap();
 
-            let loaded_public = Public::load(&example_file).unwrap();
+            // Check that the loaded value matches the original bytes
+            assert_eq!(loaded_public.deref(), &original_bytes);
 
-            // Check that the loaded public matches the original bytes
-            assert_eq!(&loaded_public.value, &original_bytes);
-
-            // Store the loaded public to a different file in the temporary directory
+            // Store the loaded value to a different file in the temporary directory
             let new_file = temp_dir.path().join("new_file");
             loaded_public.store(&new_file).unwrap();
 
@@ -224,10 +369,13 @@ mod tests {
             assert_eq!(new_file_contents, original_file_contents);
         }
 
-        /// test loading a base64 encoded public from an example file, and then storing it again in a different file
-        #[test]
-        fn test_public_load_store_base64() {
-            const N: usize = 100;
+        /// Convenience function for running a base64 load/store test
+        fn run_base64_load_store_test<
+            T: LoadValueB64<Error = anyhow::Error>
+                + StoreValueB64<Error = anyhow::Error>
+                + StoreValueB64Writer<Error = anyhow::Error>
+                + Deref<Target = [u8; N]>,
+        >() {
             // Generate original random bytes
             let original_bytes: [u8; N] = [rand::random(); N];
             // Create a temporary directory
@@ -238,9 +386,9 @@ mod tests {
             std::fs::write(&example_file, encoded_public).unwrap();
 
             // Load the public from the example file
-            let loaded_public = Public::load_b64::<{ N * 2 }, _>(&example_file).unwrap();
+            let loaded_public = T::load_b64::<{ N * 2 }, _>(&example_file).unwrap();
             // Check that the loaded public matches the original bytes
-            assert_eq!(&loaded_public.value, &original_bytes);
+            assert_eq!(loaded_public.deref(), &original_bytes);
 
             // Store the loaded public to a different file in the temporary directory
             let new_file = temp_dir.path().join("new_file");
@@ -253,7 +401,7 @@ mod tests {
             // Check that the contents of the new file match the original file
             assert_eq!(new_file_contents, original_file_contents);
 
-            //Check new file permissions are public
+            // Check new file permissions are public
             let metadata = fs::metadata(&new_file).unwrap();
             assert_eq!(metadata.permissions().mode() & 0o000777, 0o644);
 
@@ -271,9 +419,35 @@ mod tests {
             // Check that the contents of the new file match the original file
             assert_eq!(new_file_contents, original_file_contents);
 
-            //Check new file permissions are public
+            // Check new file permissions are public
             let metadata = fs::metadata(&new_file).unwrap();
             assert_eq!(metadata.permissions().mode() & 0o000777, 0o644);
         }
+
+        /// Test loading a [Public] from an example file, and then storing it again in a new file
+        #[test]
+        fn test_public_load_store() {
+            run_load_store_test::<Public<N>>();
+        }
+
+        /// Test loading a [PublicBox] from an example file, and then storing it again in a new file
+        #[test]
+        fn test_public_box_load_store() {
+            run_load_store_test::<PublicBox<N>>();
+        }
+
+        /// Test loading a base64-encoded [Public] from an example file, and then storing it again
+        /// in a different file
+        #[test]
+        fn test_public_load_store_base64() {
+            run_base64_load_store_test::<Public<N>>();
+        }
+
+        /// Test loading a base64-encoded [PublicBox] from an example file, and then storing it
+        /// again in a different file
+        #[test]
+        fn test_public_box_load_store_base64() {
+            run_base64_load_store_test::<PublicBox<N>>();
+        }
     }
 }

secret-memory/src/secret.rs

@@ -1,6 +1,5 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
-use std::convert::TryInto;
 use std::fmt;
 use std::ops::{Deref, DerefMut};
 use std::path::Path;
@@ -387,7 +386,7 @@ mod test {
 
             // Store the original secret to an example file in the temporary directory
             let example_file = temp_dir.path().join("example_file");
-            std::fs::write(example_file.clone(), &original_bytes).unwrap();
+            std::fs::write(&example_file, original_bytes).unwrap();
 
             // Load the secret from the example file
             let loaded_secret = Secret::load(&example_file).unwrap();

to/src/to/mod.rs

@@ -6,8 +6,8 @@
 //! - `Dst: ?Sized`; (e.g. [u8]) – The target to write to
 //! - `Out: Sized = &mut Dst`; (e.g. &mut [u8]) – A reference to the target to write to
 //! - `Coercable: ?Sized + DstCoercion<Dst>`; (e.g. `[u8]`, `[u8; 16]`) – Some value that
-//! destination coercion can be applied to. Usually either `Dst` itself (e.g. `[u8]` or some sized variant of
-//! `Dst` (e.g. `[u8; 64]`).
+//!   destination coercion can be applied to. Usually either `Dst` itself (e.g. `[u8]` or some sized variant of
+//!   `Dst` (e.g. `[u8; 64]`).
 //! - `Ret: Sized`; (anything) – must be `CondenseBeside<_>` if condensing is to be applied. The ordinary return value of a function with an output
 //! - `Val: Sized + BorrowMut<Dst>`; (e.g. [u8; 16]) – Some owned storage that can be borrowed as `Dst`
 //! - `Condensed: Sized = CondenseBeside<Val>::Condensed`; (e.g. [u8; 16], Result<[u8; 16]>) – The combiation of Val and Ret after condensing was applied (`Beside<Val, Ret>::condense()`/`Ret::condense(v)` for all `v : Val`).

util/Cargo.toml

@@ -16,5 +16,14 @@ base64ct = { workspace = true }
 anyhow = { workspace = true }
 typenum = { workspace = true }
 static_assertions = { workspace = true }
-rustix = {workspace = true}
-zeroize = {workspace = true}
+rustix = { workspace = true }
+zeroize = { workspace = true }
+zerocopy = { workspace = true }
+thiserror = { workspace = true }
+mio = { workspace = true }
+tempfile = { workspace = true }
+uds = { workspace = true, optional = true, features = ["mio_1xx"] }
+
+
+[features]
+experiment_file_descriptor_passing = ["uds"]

util/src/build.rs

@@ -0,0 +1,169 @@
+use crate::{
+    functional::ApplyExt,
+    mem::{SwapWithDefaultExt, SwapWithExt},
+};
+
+#[derive(thiserror::Error, Debug)]
+pub enum ConstructionSiteErectError<E> {
+    #[error("Construction site is void")]
+    IsVoid,
+    #[error("Construction is already built")]
+    AlreadyBuilt,
+    #[error("Other construction site error {0:?}")]
+    Other(#[from] E),
+}
+
+pub trait Build<T>: Sized {
+    type Error;
+    fn build(self) -> Result<T, Self::Error>;
+}
+
+#[derive(Debug)]
+pub enum ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    Void,
+    Builder(Builder),
+    Product(T),
+}
+
+impl<Builder, T> Default for ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    fn default() -> Self {
+        Self::Void
+    }
+}
+
+impl<Builder, T> ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    pub fn void() -> Self {
+        Self::Void
+    }
+
+    pub fn new(builder: Builder) -> Self {
+        Self::Builder(builder)
+    }
+
+    pub fn from_product(value: T) -> Self {
+        Self::Product(value)
+    }
+
+    pub fn take(&mut self) -> Self {
+        self.swap_with_default()
+    }
+
+    pub fn modify_taken_with_return<R, F>(&mut self, f: F) -> R
+    where
+        F: FnOnce(Self) -> (Self, R),
+    {
+        let (site, res) = self.take().apply(f);
+        self.swap_with(site);
+        res
+    }
+
+    pub fn modify_taken<F>(&mut self, f: F)
+    where
+        F: FnOnce(Self) -> Self,
+    {
+        self.take().apply(f).swap_with_mut(self)
+    }
+
+    #[allow(clippy::result_unit_err)]
+    pub fn erect(&mut self) -> Result<(), ConstructionSiteErectError<Builder::Error>> {
+        self.modify_taken_with_return(|site| {
+            let builder = match site {
+                site @ Self::Void => return (site, Err(ConstructionSiteErectError::IsVoid)),
+                site @ Self::Product(_) => {
+                    return (site, Err(ConstructionSiteErectError::AlreadyBuilt))
+                }
+                Self::Builder(builder) => builder,
+            };
+
+            let product = match builder.build() {
+                Err(e) => {
+                    return (Self::void(), Err(ConstructionSiteErectError::Other(e)));
+                }
+                Ok(p) => p,
+            };
+
+            (Self::from_product(product), Ok(()))
+        })
+    }
+
+    /// Returns `true` if the construction site is [`Void`].
+    ///
+    /// [`Void`]: ConstructionSite::Void
+    #[must_use]
+    pub fn is_void(&self) -> bool {
+        matches!(self, Self::Void)
+    }
+
+    /// Returns `true` if the construction site is [`InProgress`].
+    ///
+    /// [`InProgress`]: ConstructionSite::InProgress
+    #[must_use]
+    pub fn in_progess(&self) -> bool {
+        matches!(self, Self::Builder(..))
+    }
+
+    /// Returns `true` if the construction site is [`Done`].
+    ///
+    /// [`Done`]: ConstructionSite::Done
+    #[must_use]
+    pub fn is_available(&self) -> bool {
+        matches!(self, Self::Product(..))
+    }
+
+    pub fn into_builder(self) -> Option<Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn builder_ref(&self) -> Option<&Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn builder_mut(&mut self) -> Option<&mut Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn into_product(self) -> Option<T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn product_ref(&self) -> Option<&T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn product_mut(&mut self) -> Option<&mut T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+}

util/src/controlflow.rs

@@ -0,0 +1,149 @@
+/// A collection of control flow utility macros
+
+#[macro_export]
+/// A simple for loop to repeat a $body a number of times
+macro_rules! repeat {
+    ($times:expr, $body:expr) => {
+        for _ in 0..($times) {
+            $body
+        }
+    };
+}
+
+#[macro_export]
+/// Return unless the condition $cond is true, with return value $val, if given.
+macro_rules! return_unless {
+    ($cond:expr) => {
+        if !($cond) {
+            return;
+        }
+    };
+    ($cond:expr, $val:expr) => {
+        if !($cond) {
+            return $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Return if the condition $cond is true, with return value $val, if given.
+macro_rules! return_if {
+    ($cond:expr) => {
+        if $cond {
+            return;
+        }
+    };
+    ($cond:expr, $val:expr) => {
+        if $cond {
+            return $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Break unless the condition is true, from the loop with label $val, if given.
+macro_rules! break_if {
+    ($cond:expr) => {
+        if $cond {
+            break;
+        }
+    };
+    ($cond:expr, $val:tt) => {
+        if $cond {
+            break $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Continue if the condition is true, in the loop with label $val, if given.
+macro_rules! continue_if {
+    ($cond:expr) => {
+        if $cond {
+            continue;
+        }
+    };
+    ($cond:expr, $val:tt) => {
+        if $cond {
+            continue $val;
+        }
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_repeat() {
+        let mut sum = 0;
+        repeat!(10, {
+            sum += 1;
+        });
+        assert_eq!(sum, 10);
+    }
+
+    #[test]
+    fn test_return_unless() {
+        fn test_fn() -> i32 {
+            return_unless!(true, 1);
+            0
+        }
+        assert_eq!(test_fn(), 0);
+
+        fn test_fn2() -> i32 {
+            return_unless!(false, 1);
+            0
+        }
+        assert_eq!(test_fn2(), 1);
+    }
+
+    #[test]
+    fn test_return_if() {
+        fn test_fn() -> i32 {
+            return_if!(true, 1);
+            0
+        }
+        assert_eq!(test_fn(), 1);
+
+        fn test_fn2() -> i32 {
+            return_if!(false, 1);
+            0
+        }
+        assert_eq!(test_fn2(), 0);
+    }
+
+    #[test]
+    fn test_break_if() {
+        let mut sum = 0;
+        for i in 0..10 {
+            break_if!(i == 5);
+            sum += 1;
+        }
+        assert_eq!(sum, 5);
+
+        let mut sum = 0;
+        'one: for _ in 0..10 {
+            for j in 0..20 {
+                break_if!(j == 5, 'one);
+                sum += 1;
+            }
+        }
+        assert_eq!(sum, 5);
+    }
+
+    #[test]
+    fn test_continue_if() {
+        let mut sum = 0;
+        for i in 0..10 {
+            continue_if!(i == 5);
+            sum += 1;
+        }
+        assert_eq!(sum, 9);
+
+        let mut sum = 0;
+        'one: for i in 0..10 {
+            continue_if!(i == 5, 'one);
+            sum += 1;
+        }
+        assert_eq!(sum, 9);
+    }
+}

util/src/fd.rs

@@ -1,12 +1,300 @@
-use std::os::fd::{OwnedFd, RawFd};
+use anyhow::bail;
+use rustix::{
+    fd::{AsFd, BorrowedFd, FromRawFd, OwnedFd, RawFd},
+    io::fcntl_dupfd_cloexec,
+};
 
-/// Clone some file descriptor
+use crate::{mem::Forgetting, result::OkExt};
+
+/// Prepare a file descriptor for use in Rust code.
 ///
-/// If the file descriptor is invalid, an error will be raised.
-pub fn claim_fd(fd: RawFd) -> anyhow::Result<OwnedFd> {
-    use rustix::{fd::BorrowedFd, io::dup};
 
-    // This is safe since [dup] will simply raise
-    let fd = unsafe { dup(BorrowedFd::borrow_raw(fd))? };
-    Ok(fd)
+/// Checks if the file descriptor is valid and duplicates it to a new file descriptor.
+/// The old file descriptor is masked to avoid potential use after free (on file descriptor)
+/// in case the given file descriptor is still used somewhere
+pub fn claim_fd(fd: RawFd) -> rustix::io::Result<OwnedFd> {
+    let new = clone_fd_cloexec(unsafe { BorrowedFd::borrow_raw(fd) })?;
+    mask_fd(fd)?;
+    Ok(new)
+}
+
+/// Prepare a file descriptor for use in Rust code.
+///
+/// Checks if the file descriptor is valid.
+///
+/// Unlike [claim_fd], this will reuse the same file descriptor identifier instead of masking it.
+pub fn claim_fd_inplace(fd: RawFd) -> rustix::io::Result<OwnedFd> {
+    let mut new = unsafe { OwnedFd::from_raw_fd(fd) };
+    let tmp = clone_fd_cloexec(&new)?;
+    clone_fd_to_cloexec(tmp, &mut new)?;
+    Ok(new)
+}
+
+pub fn mask_fd(fd: RawFd) -> rustix::io::Result<()> {
+    // Safety: because the OwnedFd resulting from OwnedFd::from_raw_fd is wrapped in a Forgetting,
+    // it never gets dropped, meaning that fd is never closed and thus outlives the OwnedFd
+    let mut owned = Forgetting::new(unsafe { OwnedFd::from_raw_fd(fd) });
+    clone_fd_to_cloexec(open_nullfd()?, &mut owned)
+}
+
+pub fn clone_fd_cloexec<Fd: AsFd>(fd: Fd) -> rustix::io::Result<OwnedFd> {
+    const MINFD: RawFd = 3; // Avoid stdin, stdout, and stderr
+    fcntl_dupfd_cloexec(fd, MINFD)
+}
+
+#[cfg(target_os = "linux")]
+pub fn clone_fd_to_cloexec<Fd: AsFd>(fd: Fd, new: &mut OwnedFd) -> rustix::io::Result<()> {
+    use rustix::io::{dup3, DupFlags};
+    dup3(fd, new, DupFlags::CLOEXEC)
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn clone_fd_to_cloexec<Fd: AsFd>(fd: Fd, new: &mut OwnedFd) -> rustix::io::Result<()> {
+    use rustix::io::{dup2, fcntl_setfd, FdFlags};
+    dup2(&fd, new)?;
+    fcntl_setfd(&new, FdFlags::CLOEXEC)
+}
+
+/// Open a "blocked" file descriptor. I.e. a file descriptor that is neither meant for reading nor
+/// writing
+pub fn open_nullfd() -> rustix::io::Result<OwnedFd> {
+    use rustix::fs::{open, Mode, OFlags};
+    // TODO: Add tests showing that this will throw errors on use
+    open("/dev/null", OFlags::CLOEXEC, Mode::empty())
+}
+
+/// Convert low level errors into std::io::Error
+pub trait IntoStdioErr {
+    type Target;
+    fn into_stdio_err(self) -> Self::Target;
+}
+
+impl IntoStdioErr for rustix::io::Errno {
+    type Target = std::io::Error;
+
+    fn into_stdio_err(self) -> Self::Target {
+        std::io::Error::from_raw_os_error(self.raw_os_error())
+    }
+}
+
+impl<T> IntoStdioErr for rustix::io::Result<T> {
+    type Target = std::io::Result<T>;
+
+    fn into_stdio_err(self) -> Self::Target {
+        self.map_err(IntoStdioErr::into_stdio_err)
+    }
+}
+
+/// Read and write directly from a file descriptor
+pub struct FdIo<Fd: AsFd>(pub Fd);
+
+impl<Fd: AsFd> std::io::Read for FdIo<Fd> {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        rustix::io::read(&self.0, buf).into_stdio_err()
+    }
+}
+
+impl<Fd: AsFd> std::io::Write for FdIo<Fd> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        rustix::io::write(&self.0, buf).into_stdio_err()
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+pub trait StatExt {
+    fn is_socket(&self) -> bool;
+}
+
+impl StatExt for rustix::fs::Stat {
+    fn is_socket(&self) -> bool {
+        use rustix::fs::FileType;
+        let ft = FileType::from_raw_mode(self.st_mode);
+        matches!(ft, FileType::Socket)
+    }
+}
+
+pub trait TryStatExt {
+    type Error;
+    fn is_socket(&self) -> Result<bool, Self::Error>;
+}
+
+impl<T> TryStatExt for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn is_socket(&self) -> Result<bool, Self::Error> {
+        rustix::fs::fstat(self)?.is_socket().ok()
+    }
+}
+
+pub trait GetSocketType {
+    type Error;
+    fn socket_type(&self) -> Result<rustix::net::SocketType, Self::Error>;
+    fn is_datagram_socket(&self) -> Result<bool, Self::Error> {
+        use rustix::net::SocketType;
+        matches!(self.socket_type()?, SocketType::DGRAM).ok()
+    }
+    fn is_stream_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.socket_type()? == rustix::net::SocketType::STREAM)
+    }
+}
+
+impl<T> GetSocketType for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn socket_type(&self) -> Result<rustix::net::SocketType, Self::Error> {
+        rustix::net::sockopt::get_socket_type(self)
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetSocketDomain {
+    type Error;
+    fn socket_domain(&self) -> Result<rustix::net::AddressFamily, Self::Error>;
+    fn socket_address_family(&self) -> Result<rustix::net::AddressFamily, Self::Error> {
+        self.socket_domain()
+    }
+    fn is_unix_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.socket_domain()? == rustix::net::AddressFamily::UNIX)
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetSocketDomain for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn socket_domain(&self) -> Result<rustix::net::AddressFamily, Self::Error> {
+        rustix::net::sockopt::get_socket_domain(self)
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetUnixSocketType {
+    type Error;
+    fn is_unix_stream_socket(&self) -> Result<bool, Self::Error>;
+    fn demand_unix_stream_socket(&self) -> anyhow::Result<()>;
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetUnixSocketType for T
+where
+    T: GetSocketType + GetSocketDomain<Error = <T as GetSocketType>::Error>,
+    anyhow::Error: From<<T as GetSocketType>::Error>,
+{
+    type Error = <T as GetSocketType>::Error;
+
+    fn is_unix_stream_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.is_unix_socket()? && self.is_stream_socket()?)
+    }
+
+    fn demand_unix_stream_socket(&self) -> anyhow::Result<()> {
+        use rustix::net::AddressFamily as SA;
+        use rustix::net::SocketType as ST;
+        match (self.socket_domain()?, self.socket_type()?) {
+            (SA::UNIX, ST::STREAM) => Ok(()),
+            (SA::UNIX, mode) => bail!("Expected unix socket in stream mode, but mode is {mode:?}"),
+            (domain, _) => bail!("Expected unix socket, but socket domain is {domain:?} instead"),
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetSocketProtocol {
+    fn socket_protocol(&self) -> Result<Option<rustix::net::Protocol>, rustix::io::Errno>;
+    fn is_udp_socket(&self) -> Result<bool, rustix::io::Errno> {
+        self.socket_protocol()?
+            .map(|p| p == rustix::net::ipproto::UDP)
+            .unwrap_or(false)
+            .ok()
+    }
+    fn demand_udp_socket(&self) -> anyhow::Result<()> {
+        match self.socket_protocol() {
+            Ok(Some(rustix::net::ipproto::UDP)) => Ok(()),
+            Ok(Some(other_proto)) => {
+                bail!("Not a udp socket, instead socket protocol is: {other_proto:?}")
+            }
+            Ok(None) => bail!("getsockopt() returned empty value"),
+            Err(errno) => Err(errno.into_stdio_err())?,
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetSocketProtocol for T
+where
+    T: AsFd,
+{
+    fn socket_protocol(&self) -> Result<Option<rustix::net::Protocol>, rustix::io::Errno> {
+        rustix::net::sockopt::get_socket_protocol(self)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::{read_to_string, File};
+    use std::io::{Read, Write};
+    use std::os::fd::IntoRawFd;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_claim_fd() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let file = File::create(path.clone()).unwrap();
+        let fd: RawFd = file.into_raw_fd();
+        let owned_fd = claim_fd(fd).unwrap();
+        let mut file = unsafe { File::from_raw_fd(owned_fd.into_raw_fd()) };
+        file.write_all(b"Hello, World!").unwrap();
+
+        let message = read_to_string(path).unwrap();
+        assert_eq!(message, "Hello, World!");
+    }
+
+    #[test]
+    #[should_panic(expected = "fd != u32::MAX as RawFd")]
+    fn test_claim_fd_invalid_neg() {
+        let fd: RawFd = -1;
+        let _ = claim_fd(fd);
+    }
+
+    #[test]
+    #[should_panic(expected = "fd != u32::MAX as RawFd")]
+    fn test_claim_fd_invalid_max() {
+        let fd: RawFd = i64::MAX as RawFd;
+        let _ = claim_fd(fd);
+    }
+
+    #[test]
+    fn test_open_nullfd_write() {
+        let nullfd = open_nullfd().unwrap();
+        let mut file = unsafe { File::from_raw_fd(nullfd.into_raw_fd()) };
+        let res = file.write_all(b"Hello, World!");
+        assert!(res.is_err());
+        assert_eq!(
+            res.unwrap_err().to_string(),
+            "Bad file descriptor (os error 9)"
+        );
+    }
+
+    #[test]
+    fn test_open_nullfd_read() {
+        let nullfd = open_nullfd().unwrap();
+        let mut file = unsafe { File::from_raw_fd(nullfd.into_raw_fd()) };
+        let mut buffer = [0; 10];
+        let res = file.read_exact(&mut buffer);
+        assert!(res.is_err());
+        assert_eq!(res.unwrap_err().to_string(), "failed to fill whole buffer");
+    }
 }

util/src/file.rs

@@ -2,7 +2,6 @@ use anyhow::ensure;
 use std::fs::File;
 use std::io::Read;
 use std::os::unix::fs::OpenOptionsExt;
-use std::result::Result;
 use std::{fs::OpenOptions, path::Path};
 
 pub enum Visibility {
@@ -115,3 +114,96 @@ pub trait DisplayValueB64 {
 
     fn display_b64<'o>(&self, output: &'o mut [u8]) -> Result<&'o str, Self::Error>;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Write;
+    use std::os::unix::fs::PermissionsExt;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_fopen_w_public() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = fopen_w(path, Visibility::Public).unwrap();
+        file.write_all(b"test").unwrap();
+        let metadata = file.metadata().unwrap();
+        let permissions = metadata.permissions();
+        assert_eq!(permissions.mode(), 0o100644);
+    }
+
+    #[test]
+    fn test_fopen_w_secret() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = fopen_w(path, Visibility::Secret).unwrap();
+        file.write_all(b"test").unwrap();
+        let metadata = file.metadata().unwrap();
+        let permissions = metadata.permissions();
+        assert_eq!(permissions.mode(), 0o100600);
+    }
+
+    #[test]
+    fn test_fopen_r() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut contents = String::new();
+        let mut file = fopen_r(path).unwrap();
+        file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, "test");
+    }
+
+    #[test]
+    fn test_read_slice_to_end() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 4];
+        let mut file = fopen_r(path).unwrap();
+        file.read_slice_to_end(&mut buf).unwrap();
+        assert_eq!(buf, [116, 101, 115, 116]);
+    }
+
+    #[test]
+    fn test_read_exact_to_end() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 4];
+        let mut file = fopen_r(path).unwrap();
+        file.read_exact_to_end(&mut buf).unwrap();
+        assert_eq!(buf, [116, 101, 115, 116]);
+    }
+
+    #[test]
+    fn test_read_exact_to_end_to_long() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 3];
+        let mut file = fopen_r(path).unwrap();
+        let result = file.read_exact_to_end(&mut buf);
+        assert!(result.is_err());
+        assert_eq!(result.unwrap_err().to_string(), "File too long!");
+    }
+
+    #[test]
+    fn test_read_slice_to_end_to_long() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 3];
+        let mut file = fopen_r(path).unwrap();
+        let result = file.read_slice_to_end(&mut buf);
+        assert!(result.is_err());
+        assert_eq!(result.unwrap_err().to_string(), "File too long!");
+    }
+}

util/src/functional.rs

@@ -6,6 +6,32 @@ where
     v
 }
 
+pub trait MutatingExt {
+    fn mutating<F>(self, f: F) -> Self
+    where
+        F: Fn(&mut Self);
+    fn mutating_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&mut Self);
+}
+
+impl<T> MutatingExt for T {
+    fn mutating<F>(self, f: F) -> Self
+    where
+        F: Fn(&mut Self),
+    {
+        mutating(self, f)
+    }
+
+    fn mutating_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&mut Self),
+    {
+        f(self);
+        self
+    }
+}
+
 pub fn sideeffect<T, F>(v: T, f: F) -> T
 where
     F: Fn(&T),
@@ -13,3 +39,59 @@ where
     f(&v);
     v
 }
+
+pub trait SideffectExt {
+    fn sideeffect<F>(self, f: F) -> Self
+    where
+        F: Fn(&Self);
+    fn sideeffect_ref<F>(&self, f: F) -> &Self
+    where
+        F: Fn(&Self);
+    fn sideeffect_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&Self);
+}
+
+impl<T> SideffectExt for T {
+    fn sideeffect<F>(self, f: F) -> Self
+    where
+        F: Fn(&Self),
+    {
+        sideeffect(self, f)
+    }
+
+    fn sideeffect_ref<F>(&self, f: F) -> &Self
+    where
+        F: Fn(&Self),
+    {
+        f(self);
+        self
+    }
+
+    fn sideeffect_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&Self),
+    {
+        f(self);
+        self
+    }
+}
+
+pub fn run<R, F: FnOnce() -> R>(f: F) -> R {
+    f()
+}
+
+pub trait ApplyExt: Sized {
+    fn apply<R, F>(self, f: F) -> R
+    where
+        F: FnOnce(Self) -> R;
+}
+
+impl<T: Sized> ApplyExt for T {
+    fn apply<R, F>(self, f: F) -> R
+    where
+        F: FnOnce(Self) -> R,
+    {
+        f(self)
+    }
+}

util/src/io.rs

@@ -0,0 +1,180 @@
+use std::{borrow::Borrow, io};
+
+use anyhow::ensure;
+
+pub trait IoErrorKind {
+    fn io_error_kind(&self) -> io::ErrorKind;
+}
+
+impl<T: Borrow<io::Error>> IoErrorKind for T {
+    fn io_error_kind(&self) -> io::ErrorKind {
+        self.borrow().kind()
+    }
+}
+
+pub trait TryIoErrorKind {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind>;
+}
+
+impl<T: IoErrorKind> TryIoErrorKind for T {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind> {
+        Some(self.io_error_kind())
+    }
+}
+
+pub trait IoResultKindHintExt<T>: Sized {
+    type Error;
+    fn io_err_kind_hint(self) -> Result<T, (Self::Error, io::ErrorKind)>;
+}
+
+impl<T, E: IoErrorKind> IoResultKindHintExt<T> for Result<T, E> {
+    type Error = E;
+    fn io_err_kind_hint(self) -> Result<T, (E, io::ErrorKind)> {
+        self.map_err(|e| {
+            let kind = e.borrow().io_error_kind();
+            (e, kind)
+        })
+    }
+}
+
+pub trait TryIoResultKindHintExt<T>: Sized {
+    type Error;
+    fn try_io_err_kind_hint(self) -> Result<T, (Self::Error, Option<io::ErrorKind>)>;
+}
+
+impl<T, E: TryIoErrorKind> TryIoResultKindHintExt<T> for Result<T, E> {
+    type Error = E;
+    fn try_io_err_kind_hint(self) -> Result<T, (E, Option<io::ErrorKind>)> {
+        self.map_err(|e| {
+            let opt_kind = e.borrow().try_io_error_kind();
+            (e, opt_kind)
+        })
+    }
+}
+
+pub trait SubstituteForIoErrorKindExt<T>: Sized {
+    type Error;
+    fn substitute_for_ioerr_kind_with<F: FnOnce() -> T>(
+        self,
+        kind: io::ErrorKind,
+        f: F,
+    ) -> Result<T, Self::Error>;
+    fn substitute_for_ioerr_kind(self, kind: io::ErrorKind, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(kind, || v)
+    }
+
+    fn substitute_for_ioerr_interrupted_with<F: FnOnce() -> T>(
+        self,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(io::ErrorKind::Interrupted, f)
+    }
+
+    fn substitute_for_ioerr_interrupted(self, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_interrupted_with(|| v)
+    }
+
+    fn substitute_for_ioerr_wouldblock_with<F: FnOnce() -> T>(
+        self,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(io::ErrorKind::WouldBlock, f)
+    }
+
+    fn substitute_for_ioerr_wouldblock(self, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_wouldblock_with(|| v)
+    }
+}
+
+impl<T, E: TryIoErrorKind> SubstituteForIoErrorKindExt<T> for Result<T, E> {
+    type Error = E;
+
+    fn substitute_for_ioerr_kind_with<F: FnOnce() -> T>(
+        self,
+        kind: io::ErrorKind,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        match self.try_io_err_kind_hint() {
+            Ok(v) => Ok(v),
+            Err((_, Some(k))) if k == kind => Ok(f()),
+            Err((e, _)) => Err(e),
+        }
+    }
+}
+
+/// Automatically handles `std::io::ErrorKind::Interrupted`.
+///
+/// - If there is no error (i.e. on `Ok(r)`), the function will return `Ok(Some(r))`
+/// - `Interrupted` is handled internally, by retrying the IO operation
+/// - Other errors are returned as is
+pub fn handle_interrupted<R, E, F>(mut iofn: F) -> Result<Option<R>, E>
+where
+    E: TryIoErrorKind,
+    F: FnMut() -> Result<R, E>,
+{
+    use io::ErrorKind as E;
+    loop {
+        match iofn().try_io_err_kind_hint() {
+            Ok(v) => return Ok(Some(v)),
+            Err((_, Some(E::Interrupted))) => continue, // try again
+            Err((e, _)) => return Err(e),
+        };
+    }
+}
+
+/// Automatically handles `std::io::ErrorKind::{WouldBlock, Interrupted}`.
+///
+/// - If there is no error (i.e. on `Ok(r)`), the function will return `Ok(Some(r))`
+/// - `Interrupted` is handled internally, by retrying the IO operation
+/// - `WouldBlock` is handled by returning `Ok(None)`,
+/// - Other errors are returned as is
+pub fn nonblocking_handle_io_errors<R, E, F>(mut iofn: F) -> Result<Option<R>, E>
+where
+    E: TryIoErrorKind,
+    F: FnMut() -> Result<R, E>,
+{
+    use io::ErrorKind as E;
+    loop {
+        match iofn().try_io_err_kind_hint() {
+            Ok(v) => return Ok(Some(v)),
+            Err((_, Some(E::WouldBlock))) => return Ok(None), // no data to read
+            Err((_, Some(E::Interrupted))) => continue,       // try again
+            Err((e, _)) => return Err(e),
+        };
+    }
+}
+
+pub trait ReadNonblockingWithBoringErrorsHandledExt {
+    /// Convenience wrapper using [nonblocking_handle_io_errors] with [std::io::Read]
+    fn read_nonblocking_with_boring_errors_handled(
+        &mut self,
+        buf: &mut [u8],
+    ) -> io::Result<Option<usize>>;
+}
+
+impl<T: io::Read> ReadNonblockingWithBoringErrorsHandledExt for T {
+    fn read_nonblocking_with_boring_errors_handled(
+        &mut self,
+        buf: &mut [u8],
+    ) -> io::Result<Option<usize>> {
+        nonblocking_handle_io_errors(|| self.read(buf))
+    }
+}
+
+pub trait ReadExt {
+    fn read_exact_til_end(&mut self, buf: &mut [u8]) -> anyhow::Result<()>;
+}
+
+impl<T> ReadExt for T
+where
+    T: std::io::Read,
+{
+    fn read_exact_til_end(&mut self, buf: &mut [u8]) -> anyhow::Result<()> {
+        self.read_exact(buf)?;
+        ensure!(
+            self.read(&mut [0u8; 8])? == 0,
+            "Read source longer than buffer"
+        );
+        Ok(())
+    }
+}

util/src/length_prefix_encoding/decoder.rs

@@ -0,0 +1,359 @@
+use std::{borrow::BorrowMut, cmp::min, io};
+
+use thiserror::Error;
+use zeroize::Zeroize;
+
+use crate::{
+    io::{TryIoErrorKind, TryIoResultKindHintExt},
+    result::ensure_or,
+};
+
+pub const HEADER_SIZE: usize = std::mem::size_of::<u64>();
+
+#[derive(Error, Debug)]
+pub enum SanityError {
+    #[error("Offset is out of read buffer bounds")]
+    OutOfBufferBounds,
+    #[error("Offset is out of message buffer bounds")]
+    OutOfMessageBounds,
+}
+
+#[derive(Error, Debug)]
+#[error("Message too large ({msg_size} bytes) for buffer ({buf_size} bytes)")]
+pub struct MessageTooLargeError {
+    msg_size: usize,
+    buf_size: usize,
+}
+
+impl MessageTooLargeError {
+    pub fn new(msg_size: usize, buf_size: usize) -> Self {
+        Self { msg_size, buf_size }
+    }
+
+    pub fn ensure(msg_size: usize, buf_size: usize) -> Result<(), Self> {
+        let err = MessageTooLargeError { msg_size, buf_size };
+        ensure_or(msg_size <= buf_size, err)
+    }
+}
+
+#[derive(Debug)]
+pub struct ReadFromIoReturn<'a> {
+    pub bytes_read: usize,
+    pub message: Option<&'a mut [u8]>,
+}
+
+impl<'a> ReadFromIoReturn<'a> {
+    pub fn new(bytes_read: usize, message: Option<&'a mut [u8]>) -> Self {
+        Self {
+            bytes_read,
+            message,
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum ReadFromIoError {
+    #[error("Error reading from the underlying stream")]
+    IoError(#[from] io::Error),
+    #[error("Message size out of buffer bounds")]
+    MessageTooLargeError(#[from] MessageTooLargeError),
+}
+
+impl TryIoErrorKind for ReadFromIoError {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind> {
+        match self {
+            ReadFromIoError::IoError(ioe) => Some(ioe.kind()),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct LengthPrefixDecoder<Buf: BorrowMut<[u8]>> {
+    header: [u8; HEADER_SIZE],
+    buf: Buf,
+    off: usize,
+}
+
+impl<Buf: BorrowMut<[u8]>> LengthPrefixDecoder<Buf> {
+    pub fn new(buf: Buf) -> Self {
+        let header = Default::default();
+        let off = 0;
+        Self { header, buf, off }
+    }
+
+    pub fn clear(&mut self) {
+        self.zeroize()
+    }
+
+    pub fn from_parts(header: [u8; HEADER_SIZE], buf: Buf, off: usize) -> Self {
+        Self { header, buf, off }
+    }
+
+    pub fn into_parts(self) -> ([u8; HEADER_SIZE], Buf, usize) {
+        let Self { header, buf, off } = self;
+        (header, buf, off)
+    }
+
+    pub fn read_all_from_stdio<R: io::Read>(
+        &mut self,
+        mut r: R,
+    ) -> Result<&mut [u8], ReadFromIoError> {
+        use io::ErrorKind as K;
+        loop {
+            match self.read_from_stdio(&mut r).try_io_err_kind_hint() {
+                // Success (appeasing the borrow checker by calling message_mut())
+                Ok(ReadFromIoReturn {
+                    message: Some(_), ..
+                }) => break Ok(self.message_mut().unwrap().unwrap()),
+
+                // Unexpected EOF
+                Ok(ReadFromIoReturn { bytes_read: 0, .. }) => {
+                    break Err(ReadFromIoError::IoError(io::Error::new(
+                        K::UnexpectedEof,
+                        "",
+                    )))
+                }
+
+                // Retry
+                Ok(ReadFromIoReturn { message: None, .. }) => continue,
+                Err((_, Some(K::Interrupted))) => continue,
+
+                // Other error
+                Err((e, _)) => break Err(e),
+            }
+        }
+    }
+
+    pub fn read_from_stdio<R: io::Read>(
+        &mut self,
+        mut r: R,
+    ) -> Result<ReadFromIoReturn, ReadFromIoError> {
+        Ok(match self.next_slice_to_write_to()? {
+            // Read some bytes; any MessageTooLargeError in the call to self.message_mut() is
+            // ignored to ensure this function changes no state upon errors; the user should rerun
+            // the function and collect the MessageTooLargeError on the following invocation
+            Some(buf) => {
+                let bytes_read = r.read(buf)?;
+                self.advance(bytes_read).unwrap();
+                let message = self.message_mut().ok().flatten();
+                ReadFromIoReturn {
+                    bytes_read,
+                    message,
+                }
+            }
+            // Message is already fully read; full delegation to self.message_mut()
+            None => ReadFromIoReturn {
+                bytes_read: 0,
+                message: self.message_mut()?,
+            },
+        })
+    }
+
+    pub fn next_slice_to_write_to(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        fn some_if_nonempty(buf: &mut [u8]) -> Option<&mut [u8]> {
+            match buf.is_empty() {
+                true => None,
+                false => Some(buf),
+            }
+        }
+
+        macro_rules! return_if_nonempty_some {
+            ($opt:expr) => {{
+                // Deliberate double expansion of $opt to appease the borrow checker *sigh*
+                if $opt.and_then(some_if_nonempty).is_some() {
+                    return Ok($opt);
+                }
+            }};
+        }
+
+        return_if_nonempty_some!(Some(self.header_buffer_left_mut()));
+        return_if_nonempty_some!(self.message_fragment_left_mut()?);
+        Ok(None)
+    }
+
+    pub fn advance(&mut self, count: usize) -> Result<(), SanityError> {
+        let off = self.off + count;
+        let msg_off = off.saturating_sub(HEADER_SIZE);
+
+        use SanityError as E;
+        let alloc = self.message_buffer().len();
+        let msgsz = self.message_size();
+        ensure_or(msg_off <= alloc, E::OutOfBufferBounds)?;
+        ensure_or(
+            msgsz.map(|s| msg_off <= s).unwrap_or(true),
+            E::OutOfMessageBounds,
+        )?;
+
+        self.off = off;
+        Ok(())
+    }
+
+    pub fn ensure_sufficient_msg_buffer(&self) -> Result<(), MessageTooLargeError> {
+        let buf_size = self.message_buffer().len();
+        let msg_size = match self.get_header() {
+            None => return Ok(()),
+            Some(v) => v,
+        };
+        MessageTooLargeError::ensure(msg_size, buf_size)
+    }
+
+    pub fn header_buffer(&self) -> &[u8] {
+        &self.header[..]
+    }
+
+    pub fn header_buffer_mut(&mut self) -> &mut [u8] {
+        &mut self.header[..]
+    }
+
+    pub fn message_buffer(&self) -> &[u8] {
+        self.buf.borrow()
+    }
+
+    pub fn message_buffer_mut(&mut self) -> &mut [u8] {
+        self.buf.borrow_mut()
+    }
+
+    pub fn bytes_read(&self) -> &usize {
+        &self.off
+    }
+
+    pub fn into_message_buffer(self) -> Buf {
+        let Self { buf, .. } = self;
+        buf
+    }
+
+    pub fn header_buffer_offset(&self) -> usize {
+        min(self.off, HEADER_SIZE)
+    }
+
+    pub fn message_buffer_offset(&self) -> usize {
+        self.off.saturating_sub(HEADER_SIZE)
+    }
+
+    pub fn has_header(&self) -> bool {
+        self.header_buffer_offset() == HEADER_SIZE
+    }
+
+    pub fn has_message(&self) -> Result<bool, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        let msg_size = match self.get_header() {
+            None => return Ok(false),
+            Some(v) => v,
+        };
+        Ok(self.message_buffer_avail().len() == msg_size)
+    }
+
+    pub fn header_buffer_avail(&self) -> &[u8] {
+        let off = self.header_buffer_offset();
+        &self.header_buffer()[..off]
+    }
+
+    pub fn header_buffer_avail_mut(&mut self) -> &mut [u8] {
+        let off = self.header_buffer_offset();
+        &mut self.header_buffer_mut()[..off]
+    }
+
+    pub fn header_buffer_left(&self) -> &[u8] {
+        let off = self.header_buffer_offset();
+        &self.header_buffer()[off..]
+    }
+
+    pub fn header_buffer_left_mut(&mut self) -> &mut [u8] {
+        let off = self.header_buffer_offset();
+        &mut self.header_buffer_mut()[off..]
+    }
+
+    pub fn message_buffer_avail(&self) -> &[u8] {
+        let off = self.message_buffer_offset();
+        &self.message_buffer()[..off]
+    }
+
+    pub fn message_buffer_avail_mut(&mut self) -> &mut [u8] {
+        let off = self.message_buffer_offset();
+        &mut self.message_buffer_mut()[..off]
+    }
+
+    pub fn message_buffer_left(&self) -> &[u8] {
+        let off = self.message_buffer_offset();
+        &self.message_buffer()[off..]
+    }
+
+    pub fn message_buffer_left_mut(&mut self) -> &mut [u8] {
+        let off = self.message_buffer_offset();
+        &mut self.message_buffer_mut()[off..]
+    }
+
+    pub fn get_header(&self) -> Option<usize> {
+        match self.header_buffer_offset() == HEADER_SIZE {
+            false => None,
+            true => Some(u64::from_le_bytes(self.header) as usize),
+        }
+    }
+
+    pub fn message_size(&self) -> Option<usize> {
+        self.get_header()
+    }
+
+    pub fn encoded_message_bytes(&self) -> Option<usize> {
+        self.message_size().map(|sz| sz + HEADER_SIZE)
+    }
+
+    pub fn message_fragment(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        Ok(self.message_size().map(|sz| &self.message_buffer()[..sz]))
+    }
+
+    pub fn message_fragment_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        Ok(self
+            .message_size()
+            .map(|sz| &mut self.message_buffer_mut()[..sz]))
+    }
+
+    pub fn message_fragment_avail(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment()
+            .map(|frag| frag.map(|frag| &frag[..off]))
+    }
+
+    pub fn message_fragment_avail_mut(
+        &mut self,
+    ) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment_mut()
+            .map(|frag| frag.map(|frag| &mut frag[..off]))
+    }
+
+    pub fn message_fragment_left(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment()
+            .map(|frag| frag.map(|frag| &frag[off..]))
+    }
+
+    pub fn message_fragment_left_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment_mut()
+            .map(|frag| frag.map(|frag| &mut frag[off..]))
+    }
+
+    pub fn message(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let sz = self.message_size();
+        self.message_fragment_avail()
+            .map(|frag_opt| frag_opt.and_then(|frag| (frag.len() == sz?).then_some(frag)))
+    }
+
+    pub fn message_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let sz = self.message_size();
+        self.message_fragment_avail_mut()
+            .map(|frag_opt| frag_opt.and_then(|frag| (frag.len() == sz?).then_some(frag)))
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> Zeroize for LengthPrefixDecoder<Buf> {
+    fn zeroize(&mut self) {
+        self.header.zeroize();
+        self.message_buffer_mut().zeroize();
+        self.off.zeroize();
+    }
+}

util/src/length_prefix_encoding/encoder.rs

@@ -0,0 +1,381 @@
+use std::{
+    borrow::{Borrow, BorrowMut},
+    cmp::min,
+    io,
+};
+
+use thiserror::Error;
+use zeroize::Zeroize;
+
+use crate::{io::IoResultKindHintExt, result::ensure_or};
+
+pub const HEADER_SIZE: usize = std::mem::size_of::<u64>();
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of buffer bounds")]
+pub struct PositionOutOfBufferBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of message bounds")]
+pub struct PositionOutOfMessageBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of header bounds")]
+pub struct PositionOutOfHeaderBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Message length is bigger than buffer length")]
+pub struct MessageTooLarge;
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum MessageLenSanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    MessageTooLarge(#[from] MessageTooLarge),
+}
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum PositionSanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    PositionOutOfBufferBounds(#[from] PositionOutOfBufferBounds),
+}
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum SanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    PositionOutOfBufferBounds(#[from] PositionOutOfBufferBounds),
+    #[error("{0:?}")]
+    MessageTooLarge(#[from] MessageTooLarge),
+}
+
+impl TryFrom<SanityError> for MessageLenSanityError {
+    type Error = PositionOutOfBufferBounds;
+
+    fn try_from(value: SanityError) -> Result<Self, Self::Error> {
+        use {MessageLenSanityError as T, SanityError as F};
+        match value {
+            F::PositionOutOfMessageBounds(e) => Ok(T::PositionOutOfMessageBounds(e)),
+            F::MessageTooLarge(e) => Ok(T::MessageTooLarge(e)),
+            F::PositionOutOfBufferBounds(e) => Err(e),
+        }
+    }
+}
+
+impl From<MessageLenSanityError> for SanityError {
+    fn from(value: MessageLenSanityError) -> Self {
+        use {MessageLenSanityError as F, SanityError as T};
+        match value {
+            F::PositionOutOfMessageBounds(e) => T::PositionOutOfMessageBounds(e),
+            F::MessageTooLarge(e) => T::MessageTooLarge(e),
+        }
+    }
+}
+
+impl From<PositionSanityError> for SanityError {
+    fn from(value: PositionSanityError) -> Self {
+        use {PositionSanityError as F, SanityError as T};
+        match value {
+            F::PositionOutOfBufferBounds(e) => T::PositionOutOfBufferBounds(e),
+            F::PositionOutOfMessageBounds(e) => T::PositionOutOfMessageBounds(e),
+        }
+    }
+}
+
+pub struct WriteToIoReturn {
+    pub bytes_written: usize,
+    pub done: bool,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct LengthPrefixEncoder<Buf: Borrow<[u8]>> {
+    buf: Buf,
+    header: [u8; HEADER_SIZE],
+    pos: usize,
+}
+
+impl<Buf: Borrow<[u8]>> LengthPrefixEncoder<Buf> {
+    pub fn from_buffer(buf: Buf) -> Self {
+        let (header, pos) = ([0u8; HEADER_SIZE], 0);
+        let mut r = Self { buf, header, pos };
+        r.clear();
+        r
+    }
+
+    pub fn from_message(msg: Buf) -> Self {
+        let mut r = Self::from_buffer(msg);
+        r.restart_write_with_new_message(r.buffer_bytes().len())
+            .unwrap();
+        r
+    }
+
+    pub fn from_short_message(msg: Buf, len: usize) -> Result<Self, MessageLenSanityError> {
+        let mut r = Self::from_message(msg);
+        r.set_message_len(len)?;
+        Ok(r)
+    }
+
+    pub fn from_parts(buf: Buf, len: usize, pos: usize) -> Result<Self, SanityError> {
+        let mut r = Self::from_buffer(buf);
+        r.set_msg_len_and_position(len, pos)?;
+        Ok(r)
+    }
+
+    pub fn into_buffer(self) -> Buf {
+        let Self { buf, .. } = self;
+        buf
+    }
+
+    pub fn into_parts(self) -> (Buf, usize, usize) {
+        let len = self.message_len();
+        let pos = self.writing_position();
+        let buf = self.into_buffer();
+        (buf, len, pos)
+    }
+
+    pub fn clear(&mut self) {
+        self.set_msg_len_and_position(0, 0).unwrap();
+        self.set_message_offset(0).unwrap();
+    }
+
+    pub fn write_all_to_stdio<W: io::Write>(&mut self, mut w: W) -> io::Result<()> {
+        use io::ErrorKind as K;
+        loop {
+            match self.write_to_stdio(&mut w).io_err_kind_hint() {
+                // Done
+                Ok(WriteToIoReturn { done: true, .. }) => break Ok(()),
+
+                // Retry
+                Ok(WriteToIoReturn { done: false, .. }) => continue,
+                Err((_, K::Interrupted)) => continue,
+
+                Err((e, _)) => break Err(e),
+            }
+        }
+    }
+
+    pub fn write_to_stdio<W: io::Write>(&mut self, mut w: W) -> io::Result<WriteToIoReturn> {
+        if self.exhausted() {
+            return Ok(WriteToIoReturn {
+                bytes_written: 0,
+                done: true,
+            });
+        }
+
+        let buf = self.next_slice_to_write();
+        let bytes_written = w.write(buf)?;
+        self.advance(bytes_written).unwrap();
+
+        let done = self.exhausted();
+        Ok(WriteToIoReturn {
+            bytes_written,
+            done,
+        })
+    }
+
+    pub fn restart_write(&mut self) {
+        self.set_writing_position(0).unwrap()
+    }
+
+    pub fn restart_write_with_new_message(
+        &mut self,
+        len: usize,
+    ) -> Result<(), MessageLenSanityError> {
+        self.set_msg_len_and_position(len, 0)
+            .map_err(|e| e.try_into().unwrap())
+    }
+
+    pub fn next_slice_to_write(&self) -> &[u8] {
+        let s = self.header_left();
+        if !s.is_empty() {
+            return s;
+        }
+
+        let s = self.message_left();
+        if !s.is_empty() {
+            return s;
+        }
+
+        &[]
+    }
+
+    pub fn exhausted(&self) -> bool {
+        self.next_slice_to_write().is_empty()
+    }
+
+    pub fn message(&self) -> &[u8] {
+        &self.buffer_bytes()[..self.message_len()]
+    }
+
+    pub fn header_written(&self) -> &[u8] {
+        &self.header()[..self.header_offset()]
+    }
+
+    pub fn header_left(&self) -> &[u8] {
+        &self.header()[self.header_offset()..]
+    }
+
+    pub fn message_written(&self) -> &[u8] {
+        &self.message()[..self.message_offset()]
+    }
+
+    pub fn message_left(&self) -> &[u8] {
+        &self.message()[self.message_offset()..]
+    }
+
+    pub fn buf(&self) -> &Buf {
+        &self.buf
+    }
+
+    pub fn buffer_bytes(&self) -> &[u8] {
+        self.buf().borrow()
+    }
+
+    pub fn decode_header(&self) -> u64 {
+        u64::from_le_bytes(self.header)
+    }
+
+    pub fn header(&self) -> &[u8; HEADER_SIZE] {
+        &self.header
+    }
+
+    pub fn message_len(&self) -> usize {
+        self.decode_header() as usize
+    }
+
+    pub fn encoded_message_bytes(&self) -> usize {
+        self.message_len() + HEADER_SIZE
+    }
+
+    pub fn writing_position(&self) -> usize {
+        self.pos
+    }
+
+    pub fn header_offset(&self) -> usize {
+        min(self.writing_position(), HEADER_SIZE)
+    }
+
+    pub fn message_offset(&self) -> usize {
+        self.writing_position().saturating_sub(HEADER_SIZE)
+    }
+
+    pub fn set_header(&mut self, header: [u8; HEADER_SIZE]) -> Result<(), MessageLenSanityError> {
+        self.offset_transaction(|t| {
+            t.header = header;
+            t.ensure_msg_in_buf_bounds()?;
+            t.ensure_pos_in_msg_bounds()?;
+            Ok(())
+        })
+    }
+
+    pub fn encode_and_set_header(&mut self, header: u64) -> Result<(), MessageLenSanityError> {
+        self.set_header(header.to_le_bytes())
+    }
+
+    pub fn set_message_len(&mut self, len: usize) -> Result<(), MessageLenSanityError> {
+        self.encode_and_set_header(len as u64)
+    }
+
+    pub fn set_writing_position(&mut self, pos: usize) -> Result<(), PositionSanityError> {
+        self.offset_transaction(|t| {
+            t.pos = pos;
+            t.ensure_pos_in_buf_bounds()?;
+            t.ensure_pos_in_msg_bounds()?;
+            Ok(())
+        })
+    }
+
+    pub fn set_header_offset(&mut self, off: usize) -> Result<(), PositionOutOfHeaderBounds> {
+        ensure_or(off <= HEADER_SIZE, PositionOutOfHeaderBounds)?;
+        self.set_writing_position(off).unwrap();
+        Ok(())
+    }
+
+    pub fn set_message_offset(&mut self, off: usize) -> Result<(), PositionSanityError> {
+        self.set_writing_position(off + HEADER_SIZE)
+    }
+
+    pub fn advance(&mut self, off: usize) -> Result<(), PositionSanityError> {
+        self.set_writing_position(self.writing_position() + off)
+    }
+
+    pub fn set_msg_len_and_position(&mut self, len: usize, pos: usize) -> Result<(), SanityError> {
+        self.pos = 0;
+        self.set_message_len(len)?;
+        self.set_writing_position(pos)?;
+        Ok(())
+    }
+
+    fn offset_transaction<E, F>(&mut self, f: F) -> Result<(), E>
+    where
+        F: FnOnce(&mut LengthPrefixEncoder<&[u8]>) -> Result<(), E>,
+    {
+        let (header, pos) = {
+            let (buf, header, pos) = (self.buffer_bytes(), self.header, self.pos);
+            let mut tmp = LengthPrefixEncoder { buf, header, pos };
+            f(&mut tmp)?;
+            Ok((tmp.header, tmp.pos))
+        }?;
+        (self.header, self.pos) = (header, pos);
+        Ok(())
+    }
+
+    fn ensure_pos_in_buf_bounds(&self) -> Result<(), PositionOutOfBufferBounds> {
+        ensure_or(
+            self.message_offset() <= self.buffer_bytes().len(),
+            PositionOutOfBufferBounds,
+        )
+    }
+
+    fn ensure_pos_in_msg_bounds(&self) -> Result<(), PositionOutOfMessageBounds> {
+        ensure_or(
+            self.message_offset() <= self.message_len(),
+            PositionOutOfMessageBounds,
+        )
+    }
+
+    fn ensure_msg_in_buf_bounds(&self) -> Result<(), MessageTooLarge> {
+        ensure_or(
+            self.message_len() <= self.buffer_bytes().len(),
+            MessageTooLarge,
+        )
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> LengthPrefixEncoder<Buf> {
+    pub fn buf_mut(&mut self) -> &mut Buf {
+        &mut self.buf
+    }
+
+    pub fn buffer_bytes_mut(&mut self) -> &mut [u8] {
+        self.buf.borrow_mut()
+    }
+
+    pub fn message_mut(&mut self) -> &mut [u8] {
+        let off = self.message_len();
+        &mut self.buffer_bytes_mut()[..off]
+    }
+
+    pub fn message_written_mut(&mut self) -> &mut [u8] {
+        let off = self.message_offset();
+        &mut self.message_mut()[..off]
+    }
+
+    pub fn message_left_mut(&mut self) -> &mut [u8] {
+        let off = self.message_offset();
+        &mut self.message_mut()[off..]
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> Zeroize for LengthPrefixEncoder<Buf> {
+    fn zeroize(&mut self) {
+        self.buffer_bytes_mut().zeroize();
+        self.header.zeroize();
+        self.pos.zeroize();
+        self.clear();
+    }
+}

util/src/length_prefix_encoding/mod.rs

@@ -0,0 +1,2 @@
+pub mod decoder;
+pub mod encoder;

util/src/lib.rs

@@ -1,11 +1,18 @@
 #![recursion_limit = "256"]
 
 pub mod b64;
+pub mod build;
+pub mod controlflow;
 pub mod fd;
 pub mod file;
 pub mod functional;
+pub mod io;
+pub mod length_prefix_encoding;
 pub mod mem;
-pub mod ord;
+pub mod mio;
+pub mod option;
 pub mod result;
 pub mod time;
 pub mod typenum;
+pub mod zerocopy;
+pub mod zeroize;

util/src/mem.rs

@@ -1,5 +1,7 @@
 use std::borrow::{Borrow, BorrowMut};
 use std::cmp::min;
+use std::mem::{forget, swap};
+use std::ops::{Deref, DerefMut};
 
 /// Concatenate two byte arrays
 // TODO: Zeroize result?
@@ -31,3 +33,120 @@ pub fn cpy_min<T: BorrowMut<[u8]> + ?Sized, F: Borrow<[u8]> + ?Sized>(src: &F, d
     let len = min(src.len(), dst.len());
     dst[..len].copy_from_slice(&src[..len]);
 }
+
+/// Wrapper type to inhibit calling [std::mem::Drop] when the underlying variable is freed
+#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Default)]
+pub struct Forgetting<T> {
+    value: Option<T>,
+}
+
+impl<T> Forgetting<T> {
+    pub fn new(value: T) -> Self {
+        let value = Some(value);
+        Self { value }
+    }
+
+    pub fn extract(mut self) -> T {
+        let mut value = None;
+        swap(&mut value, &mut self.value);
+        value.unwrap()
+    }
+}
+
+impl<T> From<T> for Forgetting<T> {
+    fn from(value: T) -> Self {
+        Self::new(value)
+    }
+}
+
+impl<T> Deref for Forgetting<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value.as_ref().unwrap()
+    }
+}
+
+impl<T> DerefMut for Forgetting<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value.as_mut().unwrap()
+    }
+}
+
+impl<T> Borrow<T> for Forgetting<T> {
+    fn borrow(&self) -> &T {
+        self.deref()
+    }
+}
+
+impl<T> BorrowMut<T> for Forgetting<T> {
+    fn borrow_mut(&mut self) -> &mut T {
+        self.deref_mut()
+    }
+}
+
+impl<T> Drop for Forgetting<T> {
+    fn drop(&mut self) {
+        let mut value = None;
+        swap(&mut self.value, &mut value);
+        forget(value)
+    }
+}
+
+pub trait DiscardResultExt {
+    fn discard_result(self);
+}
+
+impl<T> DiscardResultExt for T {
+    fn discard_result(self) {}
+}
+
+pub trait ForgetExt {
+    fn forget(self);
+}
+
+impl<T> ForgetExt for T {
+    fn forget(self) {
+        std::mem::forget(self)
+    }
+}
+
+pub trait SwapWithExt {
+    fn swap_with(&mut self, other: Self) -> Self;
+    fn swap_with_mut(&mut self, other: &mut Self);
+}
+
+impl<T> SwapWithExt for T {
+    fn swap_with(&mut self, mut other: Self) -> Self {
+        self.swap_with_mut(&mut other);
+        other
+    }
+
+    fn swap_with_mut(&mut self, other: &mut Self) {
+        std::mem::swap(self, other)
+    }
+}
+
+pub trait SwapWithDefaultExt {
+    fn swap_with_default(&mut self) -> Self;
+}
+
+impl<T: Default> SwapWithDefaultExt for T {
+    fn swap_with_default(&mut self) -> Self {
+        self.swap_with(Self::default())
+    }
+}
+
+pub trait MoveExt {
+    /// Deliberately move the value
+    ///
+    /// Usually employed to enforce an object being
+    /// dropped after use.
+    fn move_here(self) -> Self;
+}
+
+impl<T: Sized> MoveExt for T {
+    fn move_here(self) -> Self {
+        self
+    }
+}

util/src/mio/mio.rs

@@ -0,0 +1,53 @@
+use mio::net::{UnixListener, UnixStream};
+use rustix::fd::{OwnedFd, RawFd};
+
+use crate::{
+    fd::{claim_fd, claim_fd_inplace},
+    result::OkExt,
+};
+
+pub mod interest {
+    use mio::Interest;
+    pub const R: Interest = Interest::READABLE;
+    pub const W: Interest = Interest::WRITABLE;
+    pub const RW: Interest = R.add(W);
+}
+
+pub trait UnixListenerExt: Sized {
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self>;
+}
+
+impl UnixListenerExt for UnixListener {
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self> {
+        use std::os::unix::net::UnixListener as StdUnixListener;
+
+        let sock = StdUnixListener::from(claim_fd(fd)?);
+        sock.set_nonblocking(true)?;
+        Ok(UnixListener::from_std(sock))
+    }
+}
+
+pub trait UnixStreamExt: Sized {
+    fn from_fd(fd: OwnedFd) -> anyhow::Result<Self>;
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self>;
+    fn claim_fd_inplace(fd: RawFd) -> anyhow::Result<Self>;
+}
+
+impl UnixStreamExt for UnixStream {
+    fn from_fd(fd: OwnedFd) -> anyhow::Result<Self> {
+        use std::os::unix::net::UnixStream as StdUnixStream;
+        #[cfg(target_os = "linux")] // TODO: We should support this on other plattforms
+        crate::fd::GetUnixSocketType::demand_unix_stream_socket(&fd)?;
+        let sock = StdUnixStream::from(fd);
+        sock.set_nonblocking(true)?;
+        UnixStream::from_std(sock).ok()
+    }
+
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self> {
+        Self::from_fd(claim_fd(fd)?)
+    }
+
+    fn claim_fd_inplace(fd: RawFd) -> anyhow::Result<Self> {
+        Self::from_fd(claim_fd_inplace(fd)?)
+    }
+}

util/src/mio/mod.rs

@@ -0,0 +1,13 @@
+#[allow(clippy::module_inception)]
+mod mio;
+pub use mio::*;
+
+#[cfg(feature = "experiment_file_descriptor_passing")]
+mod uds_send_fd;
+#[cfg(feature = "experiment_file_descriptor_passing")]
+pub use uds_send_fd::*;
+
+#[cfg(feature = "experiment_file_descriptor_passing")]
+mod uds_recv_fd;
+#[cfg(feature = "experiment_file_descriptor_passing")]
+pub use uds_recv_fd::*;

util/src/mio/uds_recv_fd.rs

@@ -0,0 +1,123 @@
+use std::{
+    borrow::{Borrow, BorrowMut},
+    collections::VecDeque,
+    io::Read,
+    marker::PhantomData,
+    os::fd::OwnedFd,
+};
+use uds::UnixStreamExt as FdPassingExt;
+
+use crate::fd::{claim_fd_inplace, IntoStdioErr};
+
+pub struct ReadWithFileDescriptors<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    socket: BorrowSock,
+    fds: BorrowFds,
+    _sock_dummy: PhantomData<Sock>,
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+    ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    pub fn new(socket: BorrowSock, fds: BorrowFds) -> Self {
+        let _sock_dummy = PhantomData;
+        Self {
+            socket,
+            fds,
+            _sock_dummy,
+        }
+    }
+
+    pub fn into_parts(self) -> (BorrowSock, BorrowFds) {
+        let Self { socket, fds, .. } = self;
+        (socket, fds)
+    }
+
+    pub fn socket(&self) -> &Sock {
+        self.socket.borrow()
+    }
+
+    pub fn fds(&self) -> &VecDeque<OwnedFd> {
+        self.fds.borrow()
+    }
+
+    pub fn fds_mut(&mut self) -> &mut VecDeque<OwnedFd> {
+        self.fds.borrow_mut()
+    }
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+    ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: BorrowMut<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    pub fn socket_mut(&mut self) -> &mut Sock {
+        self.socket.borrow_mut()
+    }
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds> Read
+    for ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        // Calculate space for additional file descriptors
+        let have_fds_before_read = self.fds().len();
+        let free_fd_slots = MAX_FDS.saturating_sub(have_fds_before_read);
+
+        // Allocate a buffer for file descriptors
+        let mut fd_buf = [0; MAX_FDS];
+        let fd_buf = &mut fd_buf[..free_fd_slots];
+
+        // Read from the unix socket
+        let (bytes_read, fds_read) = self.socket.borrow().recv_fds(buf, fd_buf)?;
+        let fd_buf = &fd_buf[..fds_read];
+
+        // Process the file descriptors
+        let mut fd_iter = fd_buf.iter();
+
+        // Try claiming all the file descriptors
+        let mut claim_fd_result = Ok(bytes_read);
+        self.fds_mut().reserve(fd_buf.len());
+        for fd in fd_iter.by_ref() {
+            match claim_fd_inplace(*fd) {
+                Ok(owned) => self.fds_mut().push_back(owned),
+                Err(e) => {
+                    // Abort on error and pass to error handler
+                    // Note that claim_fd_inplace is responsible for closing this particular
+                    // file descriptor if claiming it fails
+                    claim_fd_result = Err(e.into_stdio_err());
+                    break;
+                }
+            }
+        }
+
+        // Return if we where able to claim all file descriptors
+        if claim_fd_result.is_ok() {
+            return claim_fd_result;
+        };
+
+        // An error occurred while claiming fds
+        self.fds_mut().truncate(have_fds_before_read); // Close fds successfully claimed
+
+        // Close the remaining fds
+        for fd in fd_iter {
+            unsafe { rustix::io::close(*fd) };
+        }
+
+        claim_fd_result
+    }
+}

util/src/mio/uds_send_fd.rs

@@ -0,0 +1,121 @@
+use rustix::fd::{AsFd, AsRawFd};
+use std::{
+    borrow::{Borrow, BorrowMut},
+    cmp::min,
+    collections::VecDeque,
+    io::Write,
+    marker::PhantomData,
+};
+use uds::UnixStreamExt as FdPassingExt;
+
+use crate::{repeat, return_if};
+
+pub struct WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    socket: BorrowSock,
+    fds: BorrowFds,
+    _sock_dummy: PhantomData<Sock>,
+    _fd_dummy: PhantomData<Fd>,
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    pub fn new(socket: BorrowSock, fds: BorrowFds) -> Self {
+        let _sock_dummy = PhantomData;
+        let _fd_dummy = PhantomData;
+        Self {
+            socket,
+            fds,
+            _sock_dummy,
+            _fd_dummy,
+        }
+    }
+
+    pub fn into_parts(self) -> (BorrowSock, BorrowFds) {
+        let Self { socket, fds, .. } = self;
+        (socket, fds)
+    }
+
+    pub fn socket(&self) -> &Sock {
+        self.socket.borrow()
+    }
+
+    pub fn fds(&self) -> &VecDeque<Fd> {
+        self.fds.borrow()
+    }
+
+    pub fn fds_mut(&mut self) -> &mut VecDeque<Fd> {
+        self.fds.borrow_mut()
+    }
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: BorrowMut<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    pub fn socket_mut(&mut self) -> &mut Sock {
+        self.socket.borrow_mut()
+    }
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> Write
+    for WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        // At least one byte of real data should be sent when sending ancillary data. -- unix(7)
+        return_if!(buf.is_empty(), Ok(0));
+
+        // The kernel constant SCM_MAX_FD defines a limit on the number of file descriptors
+        // in the array.  Attempting to  send  an  array  larger  than  this  limit  causes
+        // sendmsg(2)  to fail with the error EINVAL.  SCM_MAX_FD has the value 253 (or 255
+        // before Linux 2.6.38).
+        // -- unix(7)
+        const SCM_MAX_FD: usize = 253;
+        let buf = match self.fds().len() <= SCM_MAX_FD {
+            false => &buf[..1], // Force caller to immediately call write() again to send its data
+            true => buf,
+        };
+
+        // Allocate the buffer for the file descriptor array
+        let fd_no = min(SCM_MAX_FD, self.fds().len());
+        let mut fd_buf = [0; SCM_MAX_FD]; // My kingdom for alloca(3)
+        let fd_buf = &mut fd_buf[..fd_no];
+
+        // Fill the file descriptor array
+        for (raw, fancy) in fd_buf.iter_mut().zip(self.fds().iter()) {
+            *raw = fancy.as_fd().as_raw_fd();
+        }
+
+        // Send data and file descriptors
+        let bytes_written = self.socket().send_fds(buf, fd_buf)?;
+
+        // Drop the file descriptors from the Deque
+        repeat!(fd_no, {
+            self.fds_mut().pop_front();
+        });
+
+        Ok(bytes_written)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}

util/src/option.rs

@@ -0,0 +1,7 @@
+pub trait SomeExt: Sized {
+    fn some(self) -> Option<Self> {
+        Some(self)
+    }
+}
+
+impl<T> SomeExt for T {}

util/src/ord.rs

@@ -1,8 +0,0 @@
-// TODO remove this once std::cmp::max becomes const
-pub const fn max_usize(a: usize, b: usize) -> usize {
-    if a > b {
-        a
-    } else {
-        b
-    }
-}

util/src/result.rs

@@ -1,5 +1,4 @@
 use std::convert::Infallible;
-use std::result::Result;
 
 /// Try block basically…returns a result and allows the use of the question mark operator inside
 #[macro_export]
@@ -9,6 +8,16 @@ macro_rules! attempt {
     };
 }
 
+pub trait OkExt<E>: Sized {
+    fn ok(self) -> Result<Self, E>;
+}
+
+impl<T, E> OkExt<E> for T {
+    fn ok(self) -> Result<Self, E> {
+        Ok(self)
+    }
+}
+
 /// Trait for container types that guarantee successful unwrapping.
 ///
 /// The `.guaranteed()` function can be used over unwrap to show that
@@ -26,6 +35,24 @@ pub trait GuaranteedValue {
     fn guaranteed(self) -> Self::Value;
 }
 
+pub trait FinallyExt {
+    fn finally<F: FnOnce(&mut Self)>(self, f: F) -> Self;
+}
+
+impl<T, E> FinallyExt for Result<T, E> {
+    fn finally<F: FnOnce(&mut Self)>(mut self, f: F) -> Self {
+        f(&mut self);
+        self
+    }
+}
+
+impl<T> FinallyExt for Option<T> {
+    fn finally<F: FnOnce(&mut Self)>(mut self, f: F) -> Self {
+        f(&mut self);
+        self
+    }
+}
+
 /// A result type that never contains an error.
 ///
 /// This is mostly useful in generic contexts.
@@ -97,3 +124,14 @@ impl<T> GuaranteedValue for Guaranteed<T> {
         self.unwrap()
     }
 }
+
+pub fn ensure_or<E>(b: bool, err: E) -> Result<(), E> {
+    match b {
+        true => Ok(()),
+        false => Err(err),
+    }
+}
+
+pub fn bail_if<E>(b: bool, err: E) -> Result<(), E> {
+    ensure_or(!b, err)
+}

util/src/time.rs

@@ -1,20 +1,53 @@
-use std::time::{Duration, Instant};
+use std::time::Instant;
+
+/// A timebase.
+///
+/// This is a simple wrapper around `std::time::Instant` that provides a
+/// convenient way to get the seconds elapsed since the creation of the
+/// `Timebase` instance.
 
 #[derive(Clone, Debug)]
 pub struct Timebase(Instant);
 
 impl Default for Timebase {
+    // TODO: Implement new()?
     fn default() -> Self {
         Self(Instant::now())
     }
 }
 
 impl Timebase {
+    /// Returns the seconds elapsed since the creation of the `Timebase`
     pub fn now(&self) -> f64 {
         self.0.elapsed().as_secs_f64()
     }
+}
 
-    pub fn dur(&self, t: f64) -> Duration {
-        Duration::from_secs_f64(t)
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    #[test]
+    fn test_timebase() {
+        let timebase = Timebase::default();
+        let now = timebase.now();
+        assert!(now > 0.0);
+    }
+
+    #[test]
+    fn test_timebase_clone() {
+        let timebase = Timebase::default();
+        let timebase_clone = timebase.clone();
+        assert_eq!(timebase.0, timebase_clone.0);
+    }
+
+    #[test]
+    fn test_timebase_sleep() {
+        let timebase = Timebase::default();
+        sleep(Duration::from_secs(1));
+        let now = timebase.now();
+        assert!(now > 1.0);
     }
 }

util/src/typenum.rs

@@ -19,15 +19,22 @@ pub trait IntoConst<T> {
     const VALUE: T;
 }
 
+#[allow(dead_code)]
 struct ConstApplyNegSign<T: AssociatedUnsigned, Param: IntoConst<<T as AssociatedUnsigned>::Type>>(
     *const T,
     *const Param,
 );
+
+#[allow(dead_code)]
 struct ConstApplyPosSign<T: AssociatedUnsigned, Param: IntoConst<<T as AssociatedUnsigned>::Type>>(
     *const T,
     *const Param,
 );
+
+#[allow(dead_code)]
 struct ConstLshift<T, Param: IntoConst<T>, const SHIFT: i32>(*const T, *const Param); // impl IntoConst<T>
+
+#[allow(dead_code)]
 struct ConstAdd<T, Lhs: IntoConst<T>, Rhs: IntoConst<T>>(*const T, *const Lhs, *const Rhs); // impl IntoConst<T>
 
 /// Assigns an unsigned type to a signed type

util/src/zerocopy/mod.rs

@@ -0,0 +1,7 @@
+mod ref_maker;
+mod zerocopy_ref_ext;
+mod zerocopy_slice_ext;
+
+pub use ref_maker::*;
+pub use zerocopy_ref_ext::*;
+pub use zerocopy_slice_ext::*;

util/src/zerocopy/ref_maker.rs

@@ -0,0 +1,107 @@
+use std::marker::PhantomData;
+
+use anyhow::{ensure, Context};
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+use zeroize::Zeroize;
+
+use crate::zeroize::ZeroizedExt;
+
+#[derive(Clone, Copy, Debug)]
+pub struct RefMaker<B: Sized, T> {
+    buf: B,
+    _phantom_t: PhantomData<T>,
+}
+
+impl<B, T> RefMaker<B, T> {
+    pub fn new(buf: B) -> Self {
+        let _phantom_t = PhantomData;
+        Self { buf, _phantom_t }
+    }
+
+    pub const fn target_size() -> usize {
+        std::mem::size_of::<T>()
+    }
+
+    pub fn into_buf(self) -> B {
+        self.buf
+    }
+
+    pub fn buf(&self) -> &B {
+        &self.buf
+    }
+
+    pub fn buf_mut(&mut self) -> &mut B {
+        &mut self.buf
+    }
+}
+
+impl<B: ByteSlice, T> RefMaker<B, T> {
+    pub fn parse(self) -> anyhow::Result<Ref<B, T>> {
+        self.ensure_fit()?;
+        Ref::<B, T>::new(self.buf).context("Parser error!")
+    }
+
+    pub fn from_prefix_with_tail(self) -> anyhow::Result<(Self, B)> {
+        self.ensure_fit()?;
+        let (head, tail) = self.buf.split_at(Self::target_size());
+        Ok((Self::new(head), tail))
+    }
+
+    pub fn split_prefix(self) -> anyhow::Result<(Self, Self)> {
+        self.ensure_fit()?;
+        let (head, tail) = self.buf.split_at(Self::target_size());
+        Ok((Self::new(head), Self::new(tail)))
+    }
+
+    pub fn from_prefix(self) -> anyhow::Result<Self> {
+        Ok(Self::from_prefix_with_tail(self)?.0)
+    }
+
+    pub fn from_suffix_with_head(self) -> anyhow::Result<(Self, B)> {
+        self.ensure_fit()?;
+        let point = self.bytes().len() - Self::target_size();
+        let (head, tail) = self.buf.split_at(point);
+        Ok((Self::new(tail), head))
+    }
+
+    pub fn split_suffix(self) -> anyhow::Result<(Self, Self)> {
+        self.ensure_fit()?;
+        let point = self.bytes().len() - Self::target_size();
+        let (head, tail) = self.buf.split_at(point);
+        Ok((Self::new(head), Self::new(tail)))
+    }
+
+    pub fn from_suffix(self) -> anyhow::Result<Self> {
+        Ok(Self::from_suffix_with_head(self)?.0)
+    }
+
+    pub fn bytes(&self) -> &[u8] {
+        self.buf().deref()
+    }
+
+    pub fn ensure_fit(&self) -> anyhow::Result<()> {
+        let have = self.bytes().len();
+        let need = Self::target_size();
+        ensure!(
+            need <= have,
+            "Buffer is undersized at {have} bytes (need {need} bytes)!"
+        );
+        Ok(())
+    }
+}
+
+impl<B: ByteSliceMut, T> RefMaker<B, T> {
+    pub fn make_zeroized(self) -> anyhow::Result<Ref<B, T>> {
+        self.zeroized().parse()
+    }
+
+    pub fn bytes_mut(&mut self) -> &mut [u8] {
+        self.buf_mut().deref_mut()
+    }
+}
+
+impl<B: ByteSliceMut, T> Zeroize for RefMaker<B, T> {
+    fn zeroize(&mut self) {
+        self.bytes_mut().zeroize()
+    }
+}

util/src/zerocopy/zerocopy_ref_ext.rs

@@ -0,0 +1,27 @@
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+pub trait ZerocopyEmancipateExt<B, T> {
+    fn emancipate(&self) -> Ref<&[u8], T>;
+}
+
+pub trait ZerocopyEmancipateMutExt<B, T> {
+    fn emancipate_mut(&mut self) -> Ref<&mut [u8], T>;
+}
+
+impl<B, T> ZerocopyEmancipateExt<B, T> for Ref<B, T>
+where
+    B: ByteSlice,
+{
+    fn emancipate(&self) -> Ref<&[u8], T> {
+        Ref::new(self.bytes()).unwrap()
+    }
+}
+
+impl<B, T> ZerocopyEmancipateMutExt<B, T> for Ref<B, T>
+where
+    B: ByteSliceMut,
+{
+    fn emancipate_mut(&mut self) -> Ref<&mut [u8], T> {
+        Ref::new(self.bytes_mut()).unwrap()
+    }
+}

util/src/zerocopy/zerocopy_slice_ext.rs

@@ -0,0 +1,39 @@
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::RefMaker;
+
+pub trait ZerocopySliceExt: Sized + ByteSlice {
+    fn zk_ref_maker<T>(self) -> RefMaker<Self, T> {
+        RefMaker::<Self, T>::new(self)
+    }
+
+    fn zk_parse<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().parse()
+    }
+
+    fn zk_parse_prefix<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().from_prefix()?.parse()
+    }
+
+    fn zk_parse_suffix<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().from_suffix()?.parse()
+    }
+}
+
+impl<B: ByteSlice> ZerocopySliceExt for B {}
+
+pub trait ZerocopyMutSliceExt: ZerocopySliceExt + Sized + ByteSliceMut {
+    fn zk_zeroized<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().make_zeroized()
+    }
+
+    fn zk_zeroized_from_prefix<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().from_prefix()?.make_zeroized()
+    }
+
+    fn zk_zeroized_from_suffix<T>(self) -> anyhow::Result<Ref<Self, T>> {
+        self.zk_ref_maker().from_suffix()?.make_zeroized()
+    }
+}
+
+impl<B: ByteSliceMut> ZerocopyMutSliceExt for B {}

util/src/zeroize/mod.rs

@@ -0,0 +1,2 @@
+mod zeroized_ext;
+pub use zeroized_ext::*;