Revert "chore: Format all Cargo.toml files"

This reverts commit b0706354d3.

.ci/run-regression.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+iterations="$1"
+sleep_time="$2"
+
+PWD="$(pwd)"
+EXEC="$PWD/target/release/rosenpass"
+LOGS="$PWD/output/logs"
+
+mkdir -p "$LOGS"
+
+run_command() {
+  local file=$1
+  local log_file="$2"
+  ("$EXEC" exchange-config "$file" 2>&1 | tee -a "$log_file") &
+  echo $!
+}
+
+pids=()
+
+(cd output/dut && run_command "configs/dut-$iterations.toml" "$LOGS/dut.log")
+for (( x=0; x<iterations; x++ )); do
+  (cd output/ate && run_command "configs/ate-$x.toml" "$LOGS/ate-$x.log") & pids+=($!)
+done
+
+sleep "$sleep_time"
+
+lsof -i :9999 | awk 'NR!=1 {print $2}' | xargs kill
+
+for (( x=0; x<iterations; x++ )); do
+    port=$((x + 50000))
+    lsof -i :$port | awk 'NR!=1 {print $2}' | xargs kill
+done

.github/workflows/qc.yaml

@@ -110,7 +110,12 @@ jobs:
       - run: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items
 
   cargo-test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-13]
+        # - ubuntu is x86-64
+        # - macos-13 is also x86-64 architecture
     steps:
       - uses: actions/checkout@v3
       - uses: actions/cache@v3
@@ -176,8 +181,12 @@ jobs:
           cargo fuzz run fuzz_handle_msg -- -max_total_time=5
           ulimit -s 8192000 && RUST_MIN_STACK=33554432000 && cargo fuzz run fuzz_kyber_encaps -- -max_total_time=5
           cargo fuzz run fuzz_mceliece_encaps -- -max_total_time=5
-          cargo fuzz run fuzz_box_secret_alloc -- -max_total_time=5
-          cargo fuzz run fuzz_vec_secret_alloc -- -max_total_time=5
+          cargo fuzz run fuzz_box_secret_alloc_malloc -- -max_total_time=5
+          cargo fuzz run fuzz_box_secret_alloc_memfdsec -- -max_total_time=5
+          cargo fuzz run fuzz_box_secret_alloc_memfdsec_mallocfb -- -max_total_time=5
+          cargo fuzz run fuzz_vec_secret_alloc_malloc -- -max_total_time=5
+          cargo fuzz run fuzz_vec_secret_alloc_memfdsec -- -max_total_time=5
+          cargo fuzz run fuzz_vec_secret_alloc_memfdsec_mallocfb -- -max_total_time=5
 
   codecov:
     runs-on: ubuntu-latest
@@ -186,13 +195,16 @@ jobs:
       - run: rustup component add llvm-tools-preview
       - run: |
           cargo install cargo-llvm-cov || true
-          cargo llvm-cov --lcov --output-path coverage.lcov
+          cargo llvm-cov \
+            --workspace\
+            --all-features \
+            --lcov \
+            --output-path coverage.lcov
       # If using tarapulin
       #- run: cargo install cargo-tarpaulin
       #- run: cargo tarpaulin --out Xml
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.lcov
           verbose: true

.github/workflows/regressions.yml

@@ -0,0 +1,21 @@
+name: QC
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  checks: write
+  contents: read
+
+jobs:
+  multi-peer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: cargo build --bin rosenpass --release
+      - run: python misc/generate_configs.py
+      - run: chmod +x .ci/run-regression.sh
+      - run: .ci/run-regression.sh 100 20
+      - run: |
+          [ $(ls -1 output/ate/out | wc -l) -eq 100 ]

.gitignore

@@ -20,3 +20,8 @@ _markdown_*
 **/result
 **/result-*
 .direnv
+
+# Visual studio code
+.vscode
+
+/output

.mailmap

@@ -0,0 +1 @@
+Clara Engler <cve@cve.cx> <me@emilengler.com>

CONTRIBUTING.md

@@ -0,0 +1,38 @@
+**Making a new Release of Rosenpass — Cooking Recipe**
+
+If you have to change a file, do what it takes to get the change as commit on the main branch, then **start from step 0**.
+If any other issue occurs
+
+0. Make sure you are in the root directory of the project
+   - `cd "$(git rev-parse --show-toplevel)"`
+1. Make sure you locally checked out the head of the main branch
+   - `git stash --include-untracked && git checkout main && git pull`
+2. Make sure all tests pass
+   - `cargo test`
+3. Make sure the current version in `rosenpass/Cargo.toml` matches that in the [last release on GitHub](https://github.com/rosenpass/rosenpass/releases)
+   - Only normal releases count, release candidates and draft releases can be ignored
+4. Pick the kind of release that you want to make (`major`, `minor`, `patch`, `rc`, ...)
+   - See `cargo release --help` for more information on the available release types
+   - Pick `rc` if in doubt
+5. Try to release a new version
+   - `cargo release rc --package rosenpass`
+   - An issue was reported? Go fix it, start again with step 0!
+6. Actually make the release
+   - `cargo release rc --package rosenpass --execute`
+   - Tentatively wait for any interactions, such as entering ssh keys etc.
+   - You may be asked for your ssh key multiple times!
+
+**Frequently Asked Questions (FAQ)**
+
+- You have untracked files, which `cargo release` complains about?
+  - `git stash --include-untracked`
+- You cannot push to crates.io because you are not logged in?
+  - Follow the steps displayed in [`cargo login`](https://doc.rust-lang.org/cargo/commands/cargo-login.html)
+- How is the release page added to [GitHub Releases](https://github.com/rosenpass/rosenpass/releases) itself?
+  - Our CI Pipeline will create the release, once `cargo release` pushed the new version tag to the repo. The new release should pop up almost immediately in [GitHub Releases](https://github.com/rosenpass/rosenpass/releases) after the [Actions/Release](https://github.com/rosenpass/rosenpass/actions/workflows/release.yaml) pipeline started.
+- No new release pops up in the `Release` sidebar element on the [main page](https://github.com/rosenpass/rosenpass)
+  - Did you push a `rc` release? This view only shows non-draft release, but `rc` releases are considered as draft. See [Releases](https://github.com/rosenpass/rosenpass/releases) page to see all (including draft!) releases.
+- The release page was created on GitHub, but there are no assets/artifacts other than the source code tar ball/zip?
+  - The artifacts are generated and pushed automatically to the release, but this takes some time (a couple of minutes). You can check the respective CI pipeline: [Actions/Release](https://github.com/rosenpass/rosenpass/actions/workflows/release.yaml), which should start immediately after `cargo release` pushed the new release tag to the repo. The release artifacts only are added later to the release, once all jobs in bespoke pipeline finished.
+- How are the release artifacts generated, and what are they?
+  - The release artifacts are built using one Nix derivation per platform, `nix build .#release-package`. It contains both statically linked versions of `rosenpass` itself and OCI container images.

Cargo.toml

@@ -12,15 +12,11 @@ members = [
     "fuzz",
     "secret-memory",
     "rp",
-    "wireguard-broker"
-]
-
-default-members = [
-    "rosenpass",
-    "rp",
     "wireguard-broker",
 ]
 
+default-members = ["rosenpass", "rp", "wireguard-broker"]
+
 [workspace.metadata.release]
 # ensure that adding `--package` as argument to `cargo release` still creates version tags in the form of `vx.y.z`
 tag-prefix = ""
@@ -39,40 +35,54 @@ doc-comment = "0.3.3"
 base64ct = {version = "1.6.0", default-features=false}
 zeroize = "1.8.1"
 memoffset = "0.9.1"
-thiserror = "1.0.61"
+thiserror = "1.0.64"
 paste = "1.0.15"
 env_logger = "0.10.2"
 toml = "0.7.8"
 static_assertions = "1.1.0"
 allocator-api2 = "0.2.14"
-memsec = "0.6.3"
+memsec = { git="https://github.com/rosenpass/memsec.git" ,rev="aceb9baee8aec6844125bd6612f92e9a281373df", features = [ "alloc_ext", ] }
 rand = "0.8.5"
 typenum = "1.17.0"
-log = { version = "0.4.21" }
-clap = { version = "4.5.6", features = ["derive"] }
-serde = { version = "1.0.203", features = ["derive"] }
+log = { version = "0.4.22" }
+clap = { version = "4.5.18", features = ["derive"] }
+serde = { version = "1.0.210", features = ["derive"] }
 arbitrary = { version = "1.3.2", features = ["derive"] }
-anyhow = { version = "1.0.86", features = ["backtrace", "std"] }
-mio = { version = "0.8.11", features = ["net", "os-poll"] }
-oqs-sys = { version = "0.9.1", default-features = false, features = ['classic_mceliece', 'kyber']  }
+anyhow = { version = "1.0.89", features = ["backtrace", "std"] }
+mio = { version = "1.0.2", features = ["net", "os-poll"] }
+oqs-sys = { version = "0.9.1", default-features = false, features = [
+  'classic_mceliece',
+  'kyber',
+] }
 blake2 = "0.10.6"
-chacha20poly1305 = { version = "0.10.1", default-features = false, features = [ "std", "heapless" ] }
-zerocopy = { version = "0.7.34", features = ["derive"] }
+chacha20poly1305 = { version = "0.10.1", default-features = false, features = [
+  "std",
+  "heapless",
+] }
+zerocopy = { version = "0.7.35", features = ["derive"] }
 home = "0.5.9"
-derive_builder = "0.20.0"
-tokio = { version = "1.38", features = ["macros", "rt-multi-thread"] }
-postcard= {version = "1.0.8", features = ["alloc"]}
+derive_builder = "0.20.1"
+tokio = { version = "1.40", features = ["macros", "rt-multi-thread"] }
+postcard= {version = "1.0.10", features = ["alloc"]}
+libcrux = { version = "0.0.2-pre.2" }
+hex-literal = { version = "0.4.1" }
+hex = { version = "0.4.3" }
+heck = { version = "0.5.0"  }
+libc = { version = "0.2" }
+uds = { git = "https://github.com/rosenpass/uds" }
 
 #Dev dependencies
 serial_test = "3.1.1"
-tempfile="3"
-stacker = "0.1.15"
+tempfile = "3"
+stacker = "0.1.17"
 libfuzzer-sys = "0.4"
 test_bin = "0.4.0"
 criterion = "0.4.0"
 allocator-api2-tests = "0.2.15"
+procspawn = {version = "1.0.1", features= ["test-support"]}
+
 
 #Broker dependencies (might need cleanup or changes)
-wireguard-uapi = "3.0.0"
+wireguard-uapi = { version = "3.0.0", features = ["xplatform"] }
 command-fds = "0.2.3"
-rustix = { version = "0.38.27", features = ["net"] }
+rustix = { version = "0.38.37", features = ["net", "fs"] }

analysis/01_secrecy.entry.mpv

@@ -3,33 +3,12 @@
 #define SESSION_START_EVENTS 0
 #define RANDOMIZED_CALL_IDS 0
 
-
 #include "config.mpv"
 #include "prelude/basic.mpv"
 #include "crypto/key.mpv"
 #include "crypto/kem.mpv"
-
 #include "rosenpass/oracles.mpv"
 
-nounif v:seed_prec;   attacker(prepare_seed(trusted_seed( v )))/6217[hypothesis].
-nounif v:seed;        attacker(prepare_seed( v ))/6216[hypothesis].
-nounif v:seed;        attacker(rng_kem_sk( v ))/6215[hypothesis].
-nounif v:seed;        attacker(rng_key( v ))/6214[hypothesis].
-nounif v:key_prec;    attacker(prepare_key(trusted_key( v )))/6213[hypothesis].
-nounif v:kem_sk_prec; attacker(prepare_kem_sk(trusted_kem_sk( v )))/6212[hypothesis].
-nounif v:key;         attacker(prepare_key( v ))/6211[hypothesis].
-nounif v:kem_sk;      attacker(prepare_kem_sk( v ))/6210[hypothesis].
-nounif Spk:kem_sk_tmpl;
-  attacker(Creveal_kem_pk(Spk))/6110[conclusion].
-nounif sid:SessionId, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Seski:seed_tmpl, Ssptr:seed_tmpl;
-  attacker(Cinitiator( *sid, *Ssskm, *Spsk, *Sspkt, *Seski, *Ssptr ))/6109[conclusion].
-nounif sid:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Septi:seed_tmpl, Sspti:seed_tmpl, ih:InitHello_t;
-  attacker(Cinit_hello( *sid, *biscuit_no, *Ssskm, *Spsk, *Sspkt, *Septi, *Sspti, *ih ))/6108[conclusion].
-nounif rh:RespHello_t;
-  attacker(Cresp_hello( *rh ))/6107[conclusion].
-nounif Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, ic:InitConf_t;
-  attacker(Cinit_conf( *Ssskm, *Spsk, *Sspkt, *ic ))/6106[conclusion].
-
 let main = rosenpass_main.
 
 @lemma "state coherence, initiator: Initiator accepting a RespHello message implies they also generated the associated InitHello message"

analysis/02_availability.entry.mpv

@@ -10,26 +10,6 @@
 
 let main = rosenpass_main.
 
-nounif v:seed_prec;   attacker(prepare_seed(trusted_seed( v )))/6217[hypothesis].
-nounif v:seed;        attacker(prepare_seed( v ))/6216[hypothesis].
-nounif v:seed;        attacker(rng_kem_sk( v ))/6215[hypothesis].
-nounif v:seed;        attacker(rng_key( v ))/6214[hypothesis].
-nounif v:key_prec;    attacker(prepare_key(trusted_key( v )))/6213[hypothesis].
-nounif v:kem_sk_prec; attacker(prepare_kem_sk(trusted_kem_sk( v )))/6212[hypothesis].
-nounif v:key;         attacker(prepare_key( v ))/6211[hypothesis].
-nounif v:kem_sk;      attacker(prepare_kem_sk( v ))/6210[hypothesis].
-nounif Spk:kem_sk_tmpl;
-  attacker(Creveal_kem_pk(Spk))/6110[conclusion].
-nounif sid:SessionId, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Seski:seed_tmpl, Ssptr:seed_tmpl;
-  attacker(Cinitiator( *sid, *Ssskm, *Spsk, *Sspkt, *Seski, *Ssptr ))/6109[conclusion].
-nounif sid:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Septi:seed_tmpl, Sspti:seed_tmpl, ih:InitHello_t;
-  attacker(Cinit_hello( *sid, *biscuit_no, *Ssskm, *Spsk, *Sspkt, *Septi, *Sspti, *ih ))/6108[conclusion].
-nounif rh:RespHello_t;
-  attacker(Cresp_hello( *rh ))/6107[conclusion].
-nounif Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, ic:InitConf_t;
-  attacker(Cinit_conf( *Ssskm, *Spsk, *Sspkt, *ic ))/6106[conclusion].
-
-
 @lemma "non-interruptability: Adv cannot prevent a genuine InitHello message from being accepted"
 lemma ih:InitHello_t, psk:key, sski:kem_sk, sskr:kem_sk;
   event(IHRjct(ih, psk, sskr, kem_pub(sski)))

analysis/config.mpv

@@ -88,18 +88,6 @@ set verboseCompleted=VERBOSE.
 #define SES_EV(...)
 #endif
 
-#if COOKIE_EVENTS
-#define COOKIE_EV(...) __VA_ARGS__
-#else
-#define COOKIE_EV(...)
-#endif
-
-#if KEM_EVENTS
-#define KEM_EV(...) __VA_ARGS__
-#else
-#define KEM_EV(...)
-#endif
-
 
 (* TODO: Authentication timing properties *)
 (* TODO: Proof that every adversary submitted package is equivalent to one generated by the proper algorithm using different coins. This probably requires introducing an oracle that extracts the coins used and explicitly adding the notion of coins used for Packet->Packet steps and an inductive RNG notion. *)

analysis/rosenpass/oracles.mpv

@@ -41,32 +41,23 @@ restriction s:seed, p1:Atom, p2:Atom, ad1:Atom, ad2:Atom;
   event(ConsumeSeed(p1, s, ad1)) && event(ConsumeSeed(p2, s, ad2))
     ==> p1 = p2 && ad1 = ad2.
 
-letfun create_mac2(k:key, msg:bits) = prf(k,msg).
-
 #include "rosenpass/responder.macro"
 fun Cinit_conf(kem_sk_tmpl, key_tmpl, kem_pk_tmpl, InitConf_t) : Atom [data].
 CK_EV(  event OskOinit_conf(key, key). )
 MTX_EV( event ICRjct(InitConf_t, key, kem_sk, kem_pk). )
 SES_EV( event ResponderSession(InitConf_t, key). )
-KEM_EV(event Oinit_conf_KemUse(SessionId, SessionId, Atom).)
-#ifdef KEM_EVENTS
-  restriction sidi:SessionId, sidr:SessionId, ad1:Atom, ad2:Atom;
-    event(Oinit_conf_KemUse(sidi, sidr, ad1)) && event(Oinit_conf_KemUse(sidi, sidr, ad2))
-      ==> ad1 = ad2.
-#endif
 event ConsumeBiscuit(Atom, kem_sk, kem_pk, Atom).
-
-fun Ccookie(key, bits) : Atom[data].
-
-let Oinit_conf_inner(Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, ic:InitConf_t, call:Atom) =
-
+let Oinit_conf() = 
+  in(C, Cinit_conf(Ssskm, Spsk, Sspkt, ic));
+#if RANDOMIZED_CALL_IDS
+  new call:Atom;
+#else
+  call <- Cinit_conf(Ssskm, Spsk, Sspkt, ic);
+#endif
   SETUP_HANDSHAKE_STATE()
-
   eski <- kem_sk0;
   epki <- kem_pk0;
   let try_ = (
-    let InitConf(sidi, sidr, biscuit, auth) = ic in 
-    KEM_EV(event Oinit_conf_KemUse(sidi, sidr, call);)
     INITCONF_CONSUME()
     event ConsumeBiscuit(biscuit_no, sskm, spkt, call);
     CK_EV(  event OskOinit_conf(ck_rh, osk); )
@@ -82,20 +73,10 @@ let Oinit_conf_inner(Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, ic:Ini
 #endif
   ).
 
-let Oinit_conf() = 
-
-  in(C, Cinit_conf(Ssskm, Spsk, Sspkt, ic));
-  #if RANDOMIZED_CALL_IDS
-    new call:Atom;
-  #else
-    call <- Cinit_conf(Ssskm, Spsk, Sspkt, ic);
-  #endif
-
-  Oinit_conf_inner(Ssskm, Spsk, Sspkt, ic, call).
-
 restriction biscuit_no:Atom, sskm:kem_sk, spkr:kem_pk, ad1:Atom, ad2:Atom;
   event(ConsumeBiscuit(biscuit_no, sskm, spkr, ad1)) && event(ConsumeBiscuit(biscuit_no, sskm, spkr, ad2))
     ==> ad1 = ad2.
+
 // TODO: Restriction biscuit no invalidation
 
 #include "rosenpass/initiator.macro"
@@ -104,29 +85,11 @@ CK_EV(  event OskOresp_hello(key, key, key). )
 MTX_EV( event RHRjct(RespHello_t, key, kem_sk, kem_pk). )
 MTX_EV( event ICSent(RespHello_t, InitConf_t, key, kem_sk, kem_pk). )
 SES_EV( event InitiatorSession(RespHello_t, key). )
-
-KEM_EV(event Oresp_hello_KemUse(SessionId, SessionId, Atom).)
-#ifdef KEM_EVENTS
-restriction sidi:SessionId, sidr:SessionId, ad1:Atom, ad2:Atom;
-  event(Oresp_hello_KemUse(sidi, sidr, ad1)) && event(Oresp_hello_KemUse(sidi, sidr, ad2))
-    ==> ad1 = ad2.
-#endif
-
-#ifdef COOKIE_EVENTS
-COOKIE_EVENTS(Oresp_hello)
-#endif
-let Oresp_hello(HS_DECL_ARGS, C_in:channel, call:Atom) =
-  in(C_in, Cresp_hello(RespHello(sidr, =sidi, ecti, scti, biscuit, auth)));
-  in(C_in, mac2_key:key);
+let Oresp_hello(HS_DECL_ARGS) =
+  in(C, Cresp_hello(RespHello(sidr, =sidi, ecti, scti, biscuit, auth)));
   rh <- RespHello(sidr, sidi, ecti, scti, biscuit, auth);
-  #ifdef COOKIE_EVENTS
-    msg <- RH2b(rh);
-
-    COOKIE_PROCESS(Oresp_hello,
-  #endif
   /* try */ let ic = (
     ck_ini <- ck;
-      KEM_EV(event Oresp_hello_KemUse(sidi, sidr, call);)
     RESPHELLO_CONSUME()
     ck_ih <- ck;
     INITCONF_PRODUCE()
@@ -135,25 +98,14 @@ let Oresp_hello(HS_DECL_ARGS, C_in:channel, call:Atom) =
     SES_EV( event InitiatorSession(rh, osk); )
     ic
   /* success */ ) in (
-      icbits <- IC2b(ic);
-      mac <- create_mac(spkt, icbits);
-      mac2 <- create_mac2(mac2_key, mac_envelope2b(mac));
-      out(C_in, ic);
-      out(C_in, mac);
-      out(C_in, mac2)
-
+    out(C, ic)
   /* fail */ ) else (
-  #if MESSAGE_TRANSMISSION_EVENTS
+#if MESSAGE_TRANSMISSION_EVENTS
     event RHRjct(rh, psk, sski, spkr)
-  #else
-      0
-  #endif
-    )
-#ifdef COOKIE_EVENTS
-  )
 #else
-  .
+    0
 #endif
+  ).
 
 // TODO: Restriction: Biscuit no invalidation
 
@@ -164,33 +116,24 @@ MTX_EV( event IHRjct(InitHello_t, key, kem_sk, kem_pk). )
 MTX_EV( event RHSent(InitHello_t, RespHello_t, key, kem_sk, kem_pk). )
 event ConsumeSidr(SessionId, Atom).
 event ConsumeBn(Atom, kem_sk, kem_pk, Atom).
-KEM_EV(event Oinit_hello_KemUse(SessionId, SessionId, Atom).)
-
-#ifdef KEM_EVENTS
-restriction sidi:SessionId, sidr:SessionId, ad1:Atom, ad2:Atom;
-  event(Oinit_hello_KemUse(sidi, sidr, ad1)) && event(Oinit_hello_KemUse(sidi, sidr, ad2))
-    ==> ad1 = ad2.
+let Oinit_hello() = 
+  in(C, Cinit_hello(sidr, biscuit_no, Ssskm, Spsk, Sspkt, Septi, Sspti, ih));
+#if RANDOMIZED_CALL_IDS
+  new call:Atom;
+#else
+  call <- Cinit_hello(sidr, biscuit_no, Ssskm, Spsk, Sspkt, Septi, Sspti, ih);
 #endif
-
-let Oinit_hello_inner(sidm:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt: kem_sk_tmpl, Septi: seed_tmpl, Sspti: seed_tmpl, ih: InitHello_t, mac2_key:key, C_out:channel, call:Atom) = 
   // TODO: This is ugly
   let InitHello(sidi, epki, sctr, pidiC, auth) = ih in
-
   SETUP_HANDSHAKE_STATE()
-
   eski <- kem_sk0;
-
-  event ConsumeBn(biscuit_no, sskm, spkt, call);
-  event ConsumeSidr(sidr, call);
-
   epti <- rng_key(setup_seed(Septi)); // RHR4
   spti <- rng_key(setup_seed(Sspti)); // RHR5
+  event ConsumeBn(biscuit_no, sskm, spkt, call);
+  event ConsumeSidr(sidr, call);
   event ConsumeSeed(Epti, setup_seed(Septi), call);
   event ConsumeSeed(Spti, setup_seed(Sspti), call);
-  // out(C_out, spkt);
-
   let rh = (
-    KEM_EV(event Oinit_hello_KemUse(sidi, sidr, call);)
     INITHELLO_CONSUME()
     ck_ini <- ck;
     RESPHELLO_PRODUCE()
@@ -198,14 +141,7 @@ let Oinit_hello_inner(sidm:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:k
     MTX_EV( event RHSent(ih, rh, psk, sskr, spki); )
     rh
   /* success */ ) in (
-    rhbits <- RH2b(rh);
-    mac <- create_mac(spkt, rhbits);
-    
-    out(C_out, rh);
-    out(C_out, mac);
-    mac2 <- create_mac2(mac2_key, mac_envelope2b(mac));
-    out(C_out, mac2)
-
+    out(C, rh)
   /* fail */ ) else (
 #if MESSAGE_TRANSMISSION_EVENTS
     event IHRjct(ih, psk, sskr, spki)
@@ -214,18 +150,6 @@ let Oinit_hello_inner(sidm:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:k
 #endif
   ).
 
-let Oinit_hello() = 
-  in(C, Cinit_hello(sidr, biscuit_no, Ssskm, Spsk, Sspkt, Septi, Sspti, ih));
-  in(C, mac2_key:key);
-
-  #if RANDOMIZED_CALL_IDS
-    new call:Atom;
-  #else
-    call <- Cinit_hello(sidr, biscuit_no, Ssskm, Spsk, Sspkt, Septi, Sspti, ih);
-  #endif
-
-  Oinit_hello_inner(sidr, biscuit_no, Ssskm, Spsk, Sspkt, Septi, Sspti, ih, mac2_key, C, call).
-
 restriction sid:SessionId, ad1:Atom, ad2:Atom;
   event(ConsumeSidr(sid, ad1)) && event(ConsumeSidr(sid, ad2))
     ==> ad1 = ad2.
@@ -242,55 +166,27 @@ fun Cinitiator(SessionId, kem_sk_tmpl, key_tmpl, kem_pk_tmpl, seed_tmpl, seed_tm
 CK_EV( event OskOinitiator_ck(key). )
 CK_EV( event OskOinitiator(key, key, kem_sk, kem_pk, key). )
 MTX_EV( event IHSent(InitHello_t, key, kem_sk, kem_pk). )
-KEM_EV(event Oinitiator_inner_KemUse(SessionId, SessionId, Atom).)
-
-#ifdef KEM_EVENTS
-restriction sidi:SessionId, sidr:SessionId, ad1:Atom, ad2:Atom;
-  event(Oinitiator_inner_KemUse(sidi, sidr, ad1)) && event(Oinitiator_inner_KemUse(sidi, sidr, ad2))
-    ==> ad1 = ad2.
-#endif
 event ConsumeSidi(SessionId, Atom).
-
-let Oinitiator_inner(sidi: SessionId, Ssskm: kem_sk_tmpl, Spsk: key_tmpl, Sspkt: kem_sk_tmpl, Seski: seed_tmpl, Ssptr: seed_tmpl, last_cookie:key, C_out:channel, call:Atom) =
-
+let Oinitiator() =
+  in(C, Cinitiator(sidi, Ssskm, Spsk, Sspkt, Seski, Ssptr));
+#if RANDOMIZED_CALL_IDS
+  new call:Atom;
+#else
+  call <- Cinitiator(sidi, Ssskm, Spsk, Sspkt, Seski, Ssptr);
+#endif
   SETUP_HANDSHAKE_STATE()
-    sidr <- sid0;
-
-    KEM_EV(event Oinitiator_inner_KemUse(sidi, sidr, call);)
-
   RNG_KEM_PAIR(eski, epki, Seski) // IHI3
+  sidr <- sid0;
   sptr <- rng_key(setup_seed(Ssptr)); // IHI5
   event ConsumeSidi(sidi, call);
   event ConsumeSeed(Sptr, setup_seed(Ssptr), call);
   event ConsumeSeed(Eski, setup_seed(Seski), call);
-
   INITHELLO_PRODUCE()
   CK_EV( event OskOinitiator_ck(ck); ) 
   CK_EV( event OskOinitiator(ck, psk, sski, spkr, sptr); )
   MTX_EV( event IHSent(ih, psk, sski, spkr); )
-  
-    out(C_out, ih);
-    ihbits <- IH2b(ih);
-    mac <- create_mac(spkt, ihbits);
-    out(C_out, mac);
-    mac2 <- create_mac2(last_cookie, mac_envelope2b(mac));
-    out(C_out, mac2);
-
-    Oresp_hello(HS_PASS_ARGS, C_out, call).
-
-let Oinitiator() =
-
-  in(C, Cinitiator(sidi, Ssskm, Spsk, Sspkt, Seski, Ssptr));
-
-  #if RANDOMIZED_CALL_IDS
-    new call:Atom;
-  #else
-    call <- Cinitiator(sidi, Ssskm, Spsk, Sspkt, Seski, Ssptr);
-  #endif
-
-  in(C, last_cookie:key);
-  Oinitiator_inner(sidi, Ssskm, Spsk, Sspkt, Seski, Ssptr, last_cookie, C, call).
-
+  out(C, ih);
+  Oresp_hello(HS_PASS_ARGS).
 
 restriction sid:SessionId, ad1:Atom, ad2:Atom;
   event(ConsumeSidi(sid, ad1)) && event(ConsumeSidi(sid, ad2))
@@ -311,3 +207,21 @@ let rosenpass_main() = 0
   | REP(RESPONDER_BOUND, Oinit_hello)
   | REP(RESPONDER_BOUND, Oinit_conf).
 
+nounif v:seed_prec;   attacker(prepare_seed(trusted_seed( v )))/6217[hypothesis].
+nounif v:seed;        attacker(prepare_seed( v ))/6216[hypothesis].
+nounif v:seed;        attacker(rng_kem_sk( v ))/6215[hypothesis].
+nounif v:seed;        attacker(rng_key( v ))/6214[hypothesis].
+nounif v:key_prec;    attacker(prepare_key(trusted_key( v )))/6213[hypothesis].
+nounif v:kem_sk_prec; attacker(prepare_kem_sk(trusted_kem_sk( v )))/6212[hypothesis].
+nounif v:key;         attacker(prepare_key( v ))/6211[hypothesis].
+nounif v:kem_sk;      attacker(prepare_kem_sk( v ))/6210[hypothesis].
+nounif Spk:kem_sk_tmpl;
+  attacker(Creveal_kem_pk(Spk))/6110[conclusion].
+nounif sid:SessionId, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Seski:seed_tmpl, Ssptr:seed_tmpl;
+  attacker(Cinitiator( *sid, *Ssskm, *Spsk, *Sspkt, *Seski, *Ssptr ))/6109[conclusion].
+nounif sid:SessionId, biscuit_no:Atom, Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, Septi:seed_tmpl, Sspti:seed_tmpl, ih:InitHello_t;
+  attacker(Cinit_hello( *sid, *biscuit_no, *Ssskm, *Spsk, *Sspkt, *Septi, *Sspti, *ih ))/6108[conclusion].
+nounif rh:RespHello_t;
+  attacker(Cresp_hello( *rh ))/6107[conclusion].
+nounif Ssskm:kem_sk_tmpl, Spsk:key_tmpl, Sspkt:kem_sk_tmpl, ic:InitConf_t;
+  attacker(Cinit_conf( *Ssskm, *Spsk, *Sspkt, *ic ))/6106[conclusion].

analysis/rosenpass/protocol.mpv

@@ -2,26 +2,6 @@
 #include "crypto/kem.mpv"
 #include "rosenpass/handshake_state.mpv"
 
-fun Envelope(
-  key,
-  bits
-): bits [data].
-
-type mac_envelope_t.
-fun mac_envelope(
-  key,
-  bits
-) : mac_envelope_t.
-
-fun mac_envelope2b(mac_envelope_t) : bits [typeConverter].
-
-letfun create_mac(pk:kem_pk, payload:bits) = mac_envelope(lprf2(MAC, kem_pk2b(pk), payload), payload).
-
-fun mac_envelope_pk_test(mac_envelope_t, kem_pk) : bool
-  reduc forall pk:kem_pk, b:bits;
-    mac_envelope_pk_test(mac_envelope(prf(prf(prf(prf(key0,PROTOCOL),MAC),kem_pk2b(pk)),
-        b), b), pk) = true.
-
 type InitHello_t.
 fun InitHello(
   SessionId, // sidi
@@ -31,8 +11,6 @@ fun InitHello(
   bits       // auth
 ) : InitHello_t [data].
 
-fun IH2b(InitHello_t) : bitstring [typeConverter].
-
 #define INITHELLO_PRODUCE()       \
   ck <- lprf1(CK_INIT, kem_pk2b(spkr)); /* IHI1 */ \
   /* not handled here */                /* IHI2 */ \
@@ -63,8 +41,6 @@ fun RespHello(
     bits       // auth
 ) : RespHello_t [data].
 
-fun RH2b(RespHello_t) : bitstring [typeConverter].
-
 #define RESPHELLO_PRODUCE()                   \
   /* not handled here */           /* RHR1 */ \
   MIX2(sid2b(sidr), sid2b(sidi))   /* RHR3 */ \
@@ -91,14 +67,13 @@ fun InitConf(
     bits       // auth
 ) : InitConf_t [data].
 
-fun IC2b(InitConf_t) : bitstring [typeConverter].
-
 #define INITCONF_PRODUCE()                  \
   MIX2(sid2b(sidi), sid2b(sidr)) /* ICI3 */ \
   ENCRYPT_AND_MIX(auth, empty)   /* ICI4 */ \
   ic <- InitConf(sidi, sidr, biscuit, auth);
 
 #define INITCONF_CONSUME()                        \
+  let InitConf(sidi, sidr, biscuit, auth) = ic in \
   LOAD_BISCUIT(biscuit_no, biscuit)   /* ICR1 */    \
   ENCRYPT_AND_MIX(rh_auth, empty)     /* ICIR */    \
   ck_rh <- ck;                        /* ---- */ /* TODO: Move into oracles.mpv */ \

cipher-traits/src/kem.rs

@@ -10,8 +10,6 @@
 //! The [KEM] Trait describes the basic API offered by a Key Encapsulation
 //! Mechanism. Two implementations for it are provided, [StaticKEM] and [EphemeralKEM].
 
-use std::result::Result;
-
 /// Key Encapsulation Mechanism
 ///
 /// The KEM interface defines three operations: Key generation, key encapsulation and key

ciphers/Cargo.toml

@@ -9,6 +9,9 @@ homepage = "https://rosenpass.eu/"
 repository = "https://github.com/rosenpass/rosenpass"
 readme = "readme.md"
 
+[features]
+experiment_libcrux = ["dep:libcrux"]
+
 [dependencies]
 anyhow = { workspace = true }
 rosenpass-to = { workspace = true }
@@ -20,3 +23,4 @@ static_assertions = { workspace = true }
 zeroize = { workspace = true }
 chacha20poly1305 = { workspace = true }
 blake2 = { workspace = true }
+libcrux = { workspace = true, optional = true } 

ciphers/src/lib.rs

@@ -9,7 +9,12 @@ const_assert!(KEY_LEN == hash_domain::KEY_LEN);
 
 /// Authenticated encryption with associated data
 pub mod aead {
+    #[cfg(not(feature = "experiment_libcrux"))]
     pub use crate::subtle::chacha20poly1305_ietf::{decrypt, encrypt, KEY_LEN, NONCE_LEN, TAG_LEN};
+    #[cfg(feature = "experiment_libcrux")]
+    pub use crate::subtle::chacha20poly1305_ietf_libcrux::{
+        decrypt, encrypt, KEY_LEN, NONCE_LEN, TAG_LEN,
+    };
 }
 
 /// Authenticated encryption with associated data with a constant nonce

ciphers/src/subtle/chacha20poly1305_ietf_libcrux.rs

@@ -0,0 +1,60 @@
+use rosenpass_to::ops::copy_slice;
+use rosenpass_to::To;
+
+use zeroize::Zeroize;
+
+pub const KEY_LEN: usize = 32; // Grrrr! Libcrux, please provide me these constants.
+pub const TAG_LEN: usize = 16;
+pub const NONCE_LEN: usize = 12;
+
+#[inline]
+pub fn encrypt(
+    ciphertext: &mut [u8],
+    key: &[u8],
+    nonce: &[u8],
+    ad: &[u8],
+    plaintext: &[u8],
+) -> anyhow::Result<()> {
+    let (ciphertext, mac) = ciphertext.split_at_mut(ciphertext.len() - TAG_LEN);
+
+    use libcrux::aead as C;
+    let crux_key = C::Key::Chacha20Poly1305(C::Chacha20Key(key.try_into().unwrap()));
+    let crux_iv = C::Iv(nonce.try_into().unwrap());
+
+    copy_slice(plaintext).to(ciphertext);
+    let crux_tag = libcrux::aead::encrypt(&crux_key, ciphertext, crux_iv, ad).unwrap();
+    copy_slice(crux_tag.as_ref()).to(mac);
+
+    match crux_key {
+        C::Key::Chacha20Poly1305(mut k) => k.0.zeroize(),
+        _ => panic!(),
+    }
+
+    Ok(())
+}
+
+#[inline]
+pub fn decrypt(
+    plaintext: &mut [u8],
+    key: &[u8],
+    nonce: &[u8],
+    ad: &[u8],
+    ciphertext: &[u8],
+) -> anyhow::Result<()> {
+    let (ciphertext, mac) = ciphertext.split_at(ciphertext.len() - TAG_LEN);
+
+    use libcrux::aead as C;
+    let crux_key = C::Key::Chacha20Poly1305(C::Chacha20Key(key.try_into().unwrap()));
+    let crux_iv = C::Iv(nonce.try_into().unwrap());
+    let crux_tag = C::Tag::from_slice(mac).unwrap();
+
+    copy_slice(ciphertext).to(plaintext);
+    libcrux::aead::decrypt(&crux_key, plaintext, crux_iv, ad, &crux_tag).unwrap();
+
+    match crux_key {
+        C::Key::Chacha20Poly1305(mut k) => k.0.zeroize(),
+        _ => panic!(),
+    }
+
+    Ok(())
+}

ciphers/src/subtle/mod.rs

@@ -1,4 +1,7 @@
 pub mod blake2b;
+#[cfg(not(feature = "experiment_libcrux"))]
 pub mod chacha20poly1305_ietf;
+#[cfg(feature = "experiment_libcrux")]
+pub mod chacha20poly1305_ietf_libcrux;
 pub mod incorrect_hmac_blake2b;
 pub mod xchacha20poly1305_ietf;

constant-time/src/memcmp.rs

@@ -7,18 +7,16 @@
 ///
 /// The execution time of the function grows approx. linear with the length of the input. This is
 /// considered safe.
-///
-/// ## Tests
-/// [`tests::memcmp_runs_in_constant_time`] runs a stasticial test that the equality of the two
-/// input parameters does not correlate with the run time.
-///
-/// For discussion on how to (further) ensure the constant-time execution of this function,
-/// see <https://github.com/rosenpass/rosenpass/issues/232>
 #[inline]
 pub fn memcmp(a: &[u8], b: &[u8]) -> bool {
     a.len() == b.len() && unsafe { memsec::memeq(a.as_ptr(), b.as_ptr(), a.len()) }
 }
 
+/// [tests::memcmp_runs_in_constant_time] runs a stasticial test that the equality of the two
+/// input parameters does not correlate with the run time.
+///
+/// For discussion on how to (further) ensure the constant-time execution of this function,
+/// see <https://github.com/rosenpass/rosenpass/issues/232>
 #[cfg(all(test, feature = "constant_time_tests"))]
 mod tests {
     use super::*;

doc/rosenpass.1

@@ -108,7 +108,7 @@ Rosenpass was created by Karolin Varner, Benjamin Lipp, Wanja Zaeske,
 Marei Peischl, Stephan Ajuvo, and Lisa Schmidt.
 .Pp
 This manual page was written by
-.An Emil Engler
+.An Clara Engler
 .Sh BUGS
 The bugs are tracked at
 .Lk https://github.com/rosenpass/rosenpass/issues .

doc/rp.1

@@ -113,7 +113,7 @@ Rosenpass was created by Karolin Varner, Benjamin Lipp, Wanja Zaeske,
 Marei Peischl, Stephan Ajuvo, and Lisa Schmidt.
 .Pp
 This manual page was written by
-.An Emil Engler
+.An Clara Engler
 .Sh BUGS
 The bugs are tracked at
 .Lk https://github.com/rosenpass/rosenpass/issues .

flake.nix

@@ -411,12 +411,12 @@
             inherit (packages.proof-proverif) CRYPTOVERIF_LIB;
             inputsFrom = [ packages.default ];
             nativeBuildInputs = with pkgs; [
+              inputs.fenix.packages.${system}.complete.toolchain
               cmake # override the fakecmake from the main step above
               cargo-release
               clippy
               nodePackages.prettier
               nushell # for the .ci/gen-workflow-files.nu script
-              rustfmt
               packages.proverif-patched
             ];
           };

fuzz/Cargo.toml

@@ -4,6 +4,9 @@ version = "0.0.1"
 publish = false
 edition = "2021"
 
+[features]
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]
+
 [package.metadata]
 cargo-fuzz = true
 
@@ -48,13 +51,37 @@ test = false
 doc = false
 
 [[bin]]
-name = "fuzz_box_secret_alloc"
-path = "fuzz_targets/box_secret_alloc.rs"
+name = "fuzz_box_secret_alloc_malloc"
+path = "fuzz_targets/box_secret_alloc_malloc.rs"
 test = false
 doc = false
 
 [[bin]]
-name = "fuzz_vec_secret_alloc"
-path = "fuzz_targets/vec_secret_alloc.rs"
+name = "fuzz_vec_secret_alloc_malloc"
+path = "fuzz_targets/vec_secret_alloc_malloc.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_box_secret_alloc_memfdsec"
+path = "fuzz_targets/box_secret_alloc_memfdsec.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_vec_secret_alloc_memfdsec"
+path = "fuzz_targets/vec_secret_alloc_memfdsec.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_box_secret_alloc_memfdsec_mallocfb"
+path = "fuzz_targets/box_secret_alloc_memfdsec_mallocfb.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "fuzz_vec_secret_alloc_memfdsec_mallocfb"
+path = "fuzz_targets/vec_secret_alloc_memfdsec_mallocfb.rs"
 test = false
 doc = false

fuzz/fuzz_targets/aead_enc_into.rs

@@ -15,8 +15,7 @@ pub struct Input {
 }
 
 fuzz_target!(|input: Input| {
-    let mut ciphertext: Vec<u8> = Vec::with_capacity(input.plaintext.len() + 16);
-    ciphertext.resize(input.plaintext.len() + 16, 0);
+    let mut ciphertext = vec![0u8; input.plaintext.len() + 16];
 
     aead::encrypt(
         ciphertext.as_mut_slice(),

fuzz/fuzz_targets/box_secret_alloc_malloc.rs

@@ -0,0 +1,12 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use rosenpass_secret_memory::alloc::secret_box;
+use rosenpass_secret_memory::policy::*;
+use std::sync::Once;
+static ONCE: Once = Once::new();
+
+fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_use_only_malloc_secrets);
+    let _ = secret_box(data);
+});

fuzz/fuzz_targets/box_secret_alloc_memfdsec.rs

@@ -0,0 +1,13 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use rosenpass_secret_memory::alloc::secret_box;
+use rosenpass_secret_memory::policy::*;
+use std::sync::Once;
+
+static ONCE: Once = Once::new();
+
+fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_use_only_memfd_secrets);
+    let _ = secret_box(data);
+});

fuzz/fuzz_targets/box_secret_alloc_memfdsec_mallocfb.rs

@@ -2,7 +2,12 @@
 
 use libfuzzer_sys::fuzz_target;
 use rosenpass_secret_memory::alloc::secret_box;
+use rosenpass_secret_memory::policy::*;
+use std::sync::Once;
+
+static ONCE: Once = Once::new();
 
 fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_try_use_memfd_secrets);
     let _ = secret_box(data);
 });

fuzz/fuzz_targets/handle_msg.rs

@@ -6,11 +6,15 @@ use libfuzzer_sys::fuzz_target;
 use rosenpass::protocol::CryptoServer;
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
-use rosenpass_secret_memory::Secret;
+use rosenpass_secret_memory::policy::*;
+use rosenpass_secret_memory::{PublicBox, Secret};
+use std::sync::Once;
 
+static ONCE: Once = Once::new();
 fuzz_target!(|rx_buf: &[u8]| {
+    ONCE.call_once(secret_policy_use_only_malloc_secrets);
     let sk = Secret::from_slice(&[0; StaticKem::SK_LEN]);
-    let pk = Secret::from_slice(&[0; StaticKem::PK_LEN]);
+    let pk = PublicBox::from_slice(&[0; StaticKem::PK_LEN]);
 
     let mut cs = CryptoServer::new(sk, pk);
     let mut tx_buf = [0; 10240];

fuzz/fuzz_targets/kyber_encaps.rs

@@ -14,7 +14,7 @@ pub struct Input {
 
 fuzz_target!(|input: Input| {
     let mut ciphertext = [0u8; EphemeralKem::CT_LEN];
-    let mut shared_secret = [0u8; EphemeralKem::SK_LEN];
+    let mut shared_secret = [0u8; EphemeralKem::SHK_LEN];
 
     EphemeralKem::encaps(&mut shared_secret, &mut ciphertext, &input.pk).unwrap();
 });

fuzz/fuzz_targets/vec_secret_alloc_malloc.rs

@@ -0,0 +1,15 @@
+#![no_main]
+
+use std::sync::Once;
+
+use libfuzzer_sys::fuzz_target;
+use rosenpass_secret_memory::alloc::secret_vec;
+use rosenpass_secret_memory::policy::*;
+
+static ONCE: Once = Once::new();
+
+fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_use_only_malloc_secrets);
+    let mut vec = secret_vec();
+    vec.extend_from_slice(data);
+});

fuzz/fuzz_targets/vec_secret_alloc_memfdsec.rs

@@ -0,0 +1,15 @@
+#![no_main]
+
+use std::sync::Once;
+
+use libfuzzer_sys::fuzz_target;
+use rosenpass_secret_memory::alloc::secret_vec;
+use rosenpass_secret_memory::policy::*;
+
+static ONCE: Once = Once::new();
+
+fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_use_only_memfd_secrets);
+    let mut vec = secret_vec();
+    vec.extend_from_slice(data);
+});

fuzz/fuzz_targets/vec_secret_alloc_memfdsec_mallocfb.rs

@@ -1,9 +1,15 @@
 #![no_main]
 
+use std::sync::Once;
+
 use libfuzzer_sys::fuzz_target;
 use rosenpass_secret_memory::alloc::secret_vec;
+use rosenpass_secret_memory::policy::*;
+
+static ONCE: Once = Once::new();
 
 fuzz_target!(|data: &[u8]| {
+    ONCE.call_once(secret_policy_try_use_memfd_secrets);
     let mut vec = secret_vec();
     vec.extend_from_slice(data);
 });

manual_tests/psk_broker/peer_a.rp.config

@@ -0,0 +1,13 @@
+secret_key = "peer_a.rp.sk"
+public_key = "peer_a.rp.pk"
+listen = ["[::1]:46127"]
+verbosity = "Verbose"
+
+[api]
+listen_path = []
+listen_fd = []
+stream_fd = []
+
+[[peers]]
+public_key = "peer_b.rp.pk"
+device = "rpPskBrkTestA"

manual_tests/psk_broker/peer_b.rp.config

@@ -0,0 +1,14 @@
+secret_key = "peer_b.rp.sk"
+public_key = "peer_b.rp.pk"
+listen = []
+verbosity = "Verbose"
+
+[api]
+listen_path = []
+listen_fd = []
+stream_fd = []
+
+[[peers]]
+public_key = "peer_a.rp.pk"
+endpoint = "[::1]:46127"
+device = "rpPskBrkTestB"

manual_tests/psk_broker/run_test.sh

@@ -0,0 +1,215 @@
+#! /bin/bash
+
+set -e -o pipefail
+
+enquote() {
+  while (( "$#" > 1)); do
+    printf "%q " "$1"
+    shift
+  done
+  if (("$#" > 0)); then
+    printf "%q" "$1"
+  fi
+}
+
+CLEANUP_HOOKS=()
+hook_cleanup() {
+  local hook
+  set +e +o pipefail
+  for hook in "${CLEANUP_HOOKS[@]}"; do
+    eval "${hook}"
+  done
+}
+
+cleanup() {
+  CLEANUP_HOOKS=("$(enquote exc_with_ctx cleanup "$@")" "${CLEANUP_HOOKS[@]}")
+}
+
+cleanup_eval() {
+  cleanup eval "$*"
+}
+
+stderr() {
+  echo >&2 "$@"
+}
+
+log() {
+  local level; level="$1"; shift || fatal "USAGE: log LVL MESSAGE.."
+  stderr "[${level}]" "$@"
+}
+
+info() {
+  log "INFO" "$@"
+}
+
+debug() {
+  log "DEBUG" "$@"
+}
+
+fatal() {
+  log "FATAL" "$@"
+  exit 1
+}
+
+assert() {
+  local msg; msg="$1"; shift || fatal "USAGE: assert_cmd MESSAGE COMMAND.."
+  "$@" || fatal "${msg}"
+}
+
+abs_dir() {
+  local dir; dir="$1"; shift || fatal "USAGE: abs_dir DIR"
+  (
+    cd "${dir}"
+    pwd -P
+  )
+}
+
+exc_with_ctx() {
+  local ctx; ctx="$1"; shift || fatal "USAGE: exc_with_ctx CONTEXT COMMAND.."
+  if [[ -z "${ctx}" ]]; then
+    info '$' "$@"
+  else
+    info "${ctx}\$" "$@"
+  fi
+
+  "$@"
+}
+
+exc() {
+  exc_with_ctx "" "$@"
+}
+
+exc_eval() {
+  exc eval "$*"
+} 
+
+exc_eval_with_ctx() {
+  local ctx; ctx="$1"; shift || fatal "USAGE: exc_eval_with_ctx CONTEXT EVAL_COMMAND.."
+  exc_with_ctx "eval:${ctx}" "$*"
+} 
+
+exc_as_user() {
+  exc sudo -u "${SUDO_USER}" "$@"
+}
+
+exc_eval_as_user() {
+  exc_as_user bash -c "$*"
+}
+
+fork_eval_as_user() {
+  exc sudo -u "${SUDO_USER}" bash -c "$*" &
+  local pid; pid="$!"
+  cleanup wait "${pid}"
+  cleanup pkill -2 -P "${pid}" # Reverse ordering
+}
+
+info_success() {
+  stderr
+  stderr
+  if [[ "${SUCCESS}" = 1 ]]; then
+    stderr "  Test was a success!"
+  else
+    stderr "  !!! TEST WAS A FAILURE!!!"
+  fi
+  stderr
+}
+
+main() {
+  assert "Use as root with sudo" [ "$(id -u)" -eq 0 ]
+  assert "Use as root with sudo" [ -n "${SUDO_UID}" ]
+  assert "SUDO_UID is 0; refusing to build as root" [ "${SUDO_UID}" -ne 0 ]
+
+  cleanup info_success
+
+  trap hook_cleanup EXIT
+
+  SCRIPT="$0"
+  CFG_TEMPLATE_DIR="$(abs_dir "$(dirname "${SCRIPT}")")"
+  REPO="$(abs_dir "${CFG_TEMPLATE_DIR}/../..")"
+  BINS="${REPO}/target/debug"
+
+  # Create temp dir
+  TMP_DIR="/tmp/rosenpass-psk-broker-test-$(date +%s)-$(uuidgen)"
+  cleanup rm -rf "${TMP_DIR}"
+  exc_as_user mkdir -p "${TMP_DIR}"
+
+  # Copy config
+  CFG_DIR="${TMP_DIR}/cfg"
+  exc_as_user cp -R "${CFG_TEMPLATE_DIR}" "${CFG_DIR}"
+
+  exc umask 077
+
+  exc cd "${REPO}"
+  local build_cmd; build_cmd=(cargo build --workspace --color=always --all-features --bins --profile dev)
+  if test -e "${BINS}/rosenpass-wireguard-broker-privileged" -a -e "${BINS}/rosenpass"; then
+    info "Found the binaries rosenpass-wireguard-broker-privileged and rosenpass." \
+      "Run following commands as a regular user to recompile the binaries with the right options" \
+      "in case of an error:" '$' "${build_cmd[@]}"
+  else
+    exc_as_user "${build_cmd[@]}"
+  fi
+  exc sudo setcap CAP_NET_ADMIN=+eip "${BINS}/rosenpass-wireguard-broker-privileged"
+
+  exc cd "${CFG_DIR}"
+  exc_eval_as_user "wg genkey > peer_a.wg.sk"
+  exc_eval_as_user "wg pubkey < peer_a.wg.sk > peer_a.wg.pk"
+  exc_eval_as_user "wg genkey > peer_b.wg.sk"
+  exc_eval_as_user "wg pubkey < peer_b.wg.sk > peer_b.wg.pk"
+  exc_eval_as_user "wg genpsk > peer_a_invalid.psk"
+  exc_eval_as_user "wg genpsk > peer_b_invalid.psk"
+  exc_eval_as_user "echo $(enquote "peer = \"$(cat peer_b.wg.pk)\"") >> peer_a.rp.config"
+  exc_eval_as_user "echo $(enquote "peer = \"$(cat peer_a.wg.pk)\"") >> peer_b.rp.config"
+  exc_as_user "${BINS}"/rosenpass gen-keys peer_a.rp.config
+  exc_as_user "${BINS}"/rosenpass gen-keys peer_b.rp.config
+
+  cleanup ip l del dev rpPskBrkTestA
+  cleanup ip l del dev rpPskBrkTestB
+  exc ip l add dev rpPskBrkTestA type wireguard
+  exc ip l add dev rpPskBrkTestB type wireguard
+
+  exc wg set rpPskBrkTestA \
+    listen-port 46125 \
+    private-key peer_a.wg.sk \
+    peer "$(cat peer_b.wg.pk)" \
+      endpoint 'localhost:46126' \
+      preshared-key peer_a_invalid.psk \
+      allowed-ips fe80::2/64
+  exc wg set rpPskBrkTestB \
+    listen-port 46126 \
+    private-key peer_b.wg.sk \
+    peer "$(cat peer_a.wg.pk)" \
+      endpoint 'localhost:46125' \
+      preshared-key peer_b_invalid.psk \
+      allowed-ips fe80::1/64
+
+  exc ip l set rpPskBrkTestA up
+  exc ip l set rpPskBrkTestB up
+
+  exc ip a add fe80::1/64 dev rpPskBrkTestA
+  exc ip a add fe80::2/64 dev rpPskBrkTestB
+
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "${REPO}/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass") --psk-broker-spawn \
+      exchange-config peer_a.rp.config"
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "${REPO}/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass-wireguard-broker-socket-handler") \
+      --listen-path broker.sock"
+  fork_eval_as_user "\
+    RUST_LOG='info' \
+    PATH=$(enquote "$PWD/target/debug:${PATH}") \
+    $(enquote "${BINS}/rosenpass") --psk-broker-path broker.sock \
+      exchange-config peer_b.rp.config"
+
+  exc_as_user ping -c 2 -w 10 fe80::1%rpPskBrkTestA
+  exc_as_user ping -c 2 -w 10 fe80::2%rpPskBrkTestB
+  exc_as_user ping -c 2 -w 10 fe80::2%rpPskBrkTestA
+  exc_as_user ping -c 2 -w 10 fe80::1%rpPskBrkTestB
+
+  SUCCESS=1
+}
+
+main "$@"

misc/README.md

@@ -0,0 +1,40 @@
+# Additional files
+
+This folder contains additional files that are used in the project.
+
+## `generate_configs.py`
+
+The script is used to generate configuration files for a benchmark setup
+consisting of a device under testing (DUT) and automatic test equipment (ATE),
+basically a strong machine capable of running multiple Rosenpass instances at
+once.
+
+At the top of the script multiple variables can be set to configure the DUT IP
+address and more. Once configured you may run `python3 generate_configs.py` to
+create the configuration files.
+
+A new folder called `output/` is created containing the subfolder `dut/` and
+`ate/`. The former has to be copied on the DUT, ideally reproducible hardware
+like a Raspberry Pi, while the latter is copied to the ATE, i.e. a laptop.
+
+### Running a benchmark
+
+On the ATE a run script is required since multiple instances of `rosenpass` are
+started with different configurations in parallel. The scripts are named after
+the number of instances they start, e.g. `run-50.sh` starts 50 instances.
+
+```shell
+# on the ATE aka laptop
+cd output/ate
+./run-10.sh
+```
+
+On the DUT you start a single Rosenpass instance with the configuration matching
+the ATE number of peers.
+
+```shell
+# on the DUT aka Raspberry Pi
+rosenpass exchange-config configs/dut-10.toml
+```
+
+Use whatever measurement tool you like to monitor the DUT and ATE.

misc/generate_configs.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+from subprocess import run
+import os
+
+config = dict(
+    peer_counts=[1, 5, 10, 50, 100, 500],
+    peer_count_max=100,
+    ate_ip="127.0.0.1",
+    dut_ip="127.0.0.1",
+    dut_port=9999,
+    path_to_rosenpass_bin=os.getcwd() + "/target/release/rosenpass",
+)
+
+print(config)
+
+output_dir = Path("output")
+output_dir.mkdir(exist_ok=True)
+
+template_dut = """
+public_key = "keys/dut-public-key"
+secret_key = "keys/dut-secret-key"
+listen = ["{dut_ip}:{dut_port}"]
+verbosity = "Quiet"
+"""
+template_dut_peer = """
+[[peers]] # ATE-{i}
+public_key = "keys/ate-{i}-public-key"
+endpoint = "{ate_ip}:{ate_port}"
+key_out = "out/key_out_{i}"
+"""
+
+template_ate = """
+public_key = "keys/ate-{i}-public-key"
+secret_key = "keys/ate-{i}-secret-key"
+listen = ["{ate_ip}:{ate_port}"]
+verbosity = "Quiet"
+
+[[peers]] # DUT
+public_key = "keys/dut-public-key"
+endpoint = "{dut_ip}:{dut_port}"
+key_out = "out/key_out_{i}"
+"""
+
+(output_dir / "dut" / "keys").mkdir(exist_ok=True, parents=True)
+(output_dir / "dut" / "out").mkdir(exist_ok=True, parents=True)
+(output_dir / "dut" / "configs").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "keys").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "out").mkdir(exist_ok=True, parents=True)
+(output_dir / "ate" / "configs").mkdir(exist_ok=True, parents=True)
+
+for peer_count in config["peer_counts"]:
+    dut_config = template_dut.format(**config)
+    for i in range(peer_count):
+        dut_config += template_dut_peer.format(**config, i=i, ate_port=50000 + i)
+
+    (output_dir / "dut" / "configs" / f"dut-{peer_count}.toml").write_text(dut_config)
+
+    if not (output_dir / "dut" / "keys" / "dut-public-key").exists():
+        print("Generate DUT keys")
+        run(
+            [
+                config["path_to_rosenpass_bin"],
+                "gen-keys",
+                f"configs/dut-{peer_count}.toml",
+            ],
+            cwd=output_dir / "dut",
+        )
+    else:
+        print("DUT keys already exist")
+
+# copy the DUT public key to the ATE
+(output_dir / "ate" / "keys" / "dut-public-key").write_bytes(
+    (output_dir / "dut" / "keys" / "dut-public-key").read_bytes()
+)
+
+ate_script = "(trap 'kill 0' SIGINT; \\\n"
+
+for i in range(config["peer_count_max"]):
+    (output_dir / "ate" / "configs" / f"ate-{i}.toml").write_text(
+        template_ate.format(**config, i=i, ate_port=50000 + i)
+    )
+
+    if not (output_dir / "ate" / "keys" / f"ate-{i}-public-key").exists():
+        # generate ATE keys
+        run(
+            [config["path_to_rosenpass_bin"], "gen-keys", f"configs/ate-{i}.toml"],
+            cwd=output_dir / "ate",
+        )
+    else:
+        print(f"ATE-{i} keys already exist")
+
+    # copy the ATE public keys to the DUT
+    (output_dir / "dut" / "keys" / f"ate-{i}-public-key").write_bytes(
+        (output_dir / "ate" / "keys" / f"ate-{i}-public-key").read_bytes()
+    )
+
+    ate_script += (
+        f"{config['path_to_rosenpass_bin']} exchange-config configs/ate-{i}.toml & \\\n"
+    )
+
+    if (i + 1) in config["peer_counts"]:
+        write_script = ate_script
+        write_script += "wait)"
+
+        (output_dir / "ate" / f"run-{i+1}.sh").write_text(write_script)

readme.md

@@ -66,6 +66,8 @@ A wrapper script provides instant feedback about which queries execute as expect
 
 # Getting Rosenpass
 
+Documentation and installation guides can be found at the [Rosenpass website](https://rosenpass.eu/docs).
+
 Rosenpass is packaged for more and more distributions, maybe also for the distribution of your choice?
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/rosenpass.svg)](https://repology.org/project/rosenpass/versions)

rosenpass/Cargo.toml

@@ -13,6 +13,19 @@ readme = "readme.md"
 name = "rosenpass"
 path = "src/main.rs"
 
+[[bin]]
+name = "rosenpass-gen-ipc-msg-types"
+path = "src/bin/gen-ipc-msg-types.rs"
+required-features = ["experiment_api", "internal_bin_gen_ipc_msg_types"]
+
+[[test]]
+name = "api-integration-tests"
+required-features = ["experiment_api", "internal_testing"]
+
+[[test]]
+name = "api-integration-tests-api-setup"
+required-features = ["experiment_api", "internal_testing"]
+
 [[bench]]
 name = "handshake"
 harness = false
@@ -40,6 +53,13 @@ zerocopy = { workspace = true }
 home = { workspace = true }
 derive_builder = {workspace = true}
 rosenpass-wireguard-broker = {workspace = true}
+zeroize = { workspace = true }
+hex-literal = { workspace = true, optional = true }
+hex = { workspace = true, optional = true }
+heck = { workspace = true, optional = true }
+command-fds = { workspace = true, optional = true }
+rustix = { workspace = true }
+uds = { workspace = true, optional = true, features = ["mio_1xx"] }
 
 [build-dependencies]
 anyhow = { workspace = true }
@@ -49,6 +69,14 @@ criterion = { workspace = true }
 test_bin = { workspace = true }
 stacker = { workspace = true }
 serial_test = {workspace = true}
+procspawn = {workspace = true}
+tempfile = { workspace = true }
+rustix = {workspace = true}
 
 [features]
-enable_broker_api = ["rosenpass-wireguard-broker/enable_broker_api"]
+default = ["experiment_api"]
+experiment_memfd_secret = ["rosenpass-wireguard-broker/experiment_memfd_secret"]
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]
+experiment_api = ["hex-literal", "uds", "command-fds", "rosenpass-util/experiment_file_descriptor_passing", "rosenpass-wireguard-broker/experiment_api"]
+internal_testing  = []
+internal_bin_gen_ipc_msg_types = ["hex", "heck"]

rosenpass/benches/handshake.rs

@@ -1,10 +1,12 @@
 use anyhow::Result;
 use rosenpass::protocol::{CryptoServer, HandleMsgResult, MsgBuf, PeerPtr, SPk, SSk, SymKey};
+use std::ops::DerefMut;
 
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use rosenpass_secret_memory::secret_policy_try_use_memfd_secrets;
 
 fn handle(
     tx: &mut CryptoServer,
@@ -39,7 +41,7 @@ fn hs(ini: &mut CryptoServer, res: &mut CryptoServer) -> Result<()> {
 
 fn keygen() -> Result<(SSk, SPk)> {
     let (mut sk, mut pk) = (SSk::zero(), SPk::zero());
-    StaticKem::keygen(sk.secret_mut(), pk.secret_mut())?;
+    StaticKem::keygen(sk.secret_mut(), pk.deref_mut())?;
     Ok((sk, pk))
 }
 
@@ -56,6 +58,7 @@ fn make_server_pair() -> Result<(CryptoServer, CryptoServer)> {
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
+    secret_policy_try_use_memfd_secrets();
     let (mut a, mut b) = make_server_pair().unwrap();
     c.bench_function("cca_secret_alloc", |bench| {
         bench.iter(|| {

rosenpass/src/api/api_handler.rs

@@ -0,0 +1,295 @@
+use std::{borrow::BorrowMut, collections::VecDeque, os::fd::OwnedFd};
+
+use anyhow::Context;
+use rosenpass_to::{ops::copy_slice, To};
+use rosenpass_util::{
+    fd::FdIo,
+    functional::{run, ApplyExt},
+    io::ReadExt,
+    mem::DiscardResultExt,
+    mio::UnixStreamExt,
+    result::OkExt,
+};
+use rosenpass_wireguard_broker::brokers::mio_client::MioBrokerClient;
+
+use crate::{
+    api::{add_listen_socket_response_status, add_psk_broker_response_status},
+    app_server::AppServer,
+    protocol::BuildCryptoServer,
+};
+
+use super::{supply_keypair_response_status, Server as ApiServer};
+
+#[derive(Debug)]
+pub struct ApiHandler {
+    _dummy: (),
+}
+
+impl ApiHandler {
+    #[allow(clippy::new_without_default)]
+    pub fn new() -> Self {
+        Self { _dummy: () }
+    }
+}
+
+pub trait ApiHandlerContext {
+    fn api_handler(&self) -> &ApiHandler;
+    fn app_server(&self) -> &AppServer;
+    fn api_handler_mut(&mut self) -> &mut ApiHandler;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+}
+
+#[derive(thiserror::Error, Debug)]
+#[error("Error in SupplyKeypair")]
+struct SupplyKeypairError {
+    status: u128,
+    #[source]
+    cause: anyhow::Error,
+}
+
+trait SupplyKeypairErrorExt<T> {
+    fn e_custom(self, status: u128) -> Result<T, SupplyKeypairError>;
+    fn einternal(self) -> Result<T, SupplyKeypairError>;
+    fn ealready_supplied(self) -> Result<T, SupplyKeypairError>;
+    fn einvalid_req(self) -> Result<T, SupplyKeypairError>;
+}
+
+impl<T, E: Into<anyhow::Error>> SupplyKeypairErrorExt<T> for Result<T, E> {
+    fn e_custom(self, status: u128) -> Result<T, SupplyKeypairError> {
+        self.map_err(|e| SupplyKeypairError {
+            status,
+            cause: e.into(),
+        })
+    }
+
+    fn einternal(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::INTERNAL_ERROR)
+    }
+
+    fn ealready_supplied(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::KEYPAIR_ALREADY_SUPPLIED)
+    }
+
+    fn einvalid_req(self) -> Result<T, SupplyKeypairError> {
+        self.e_custom(supply_keypair_response_status::INVALID_REQUEST)
+    }
+}
+
+impl<T> ApiServer for T
+where
+    T: ?Sized + ApiHandlerContext,
+{
+    fn ping(
+        &mut self,
+        req: &super::PingRequest,
+        _req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::PingResponse,
+    ) -> anyhow::Result<()> {
+        let (req, res) = (&req.payload, &mut res.payload);
+        copy_slice(&req.echo).to(&mut res.echo);
+        Ok(())
+    }
+
+    fn supply_keypair(
+        &mut self,
+        req: &super::SupplyKeypairRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::SupplyKeypairResponse,
+    ) -> anyhow::Result<()> {
+        let outcome: Result<(), SupplyKeypairError> = run(|| {
+            // Acquire the file descriptors
+            let mut sk_io = FdIo(
+                req_fds
+                    .front()
+                    .context("First file descriptor, secret key, missing.")
+                    .einvalid_req()?,
+            );
+            let mut pk_io = FdIo(
+                req_fds
+                    .get(1)
+                    .context("Second file descriptor, public key, missing.")
+                    .einvalid_req()?,
+            );
+
+            // Actually read the secrets
+            let mut sk = crate::protocol::SSk::zero();
+            sk_io.read_exact_til_end(sk.secret_mut()).einvalid_req()?;
+
+            let mut pk = crate::protocol::SPk::zero();
+            pk_io.read_exact_til_end(pk.borrow_mut()).einvalid_req()?;
+
+            // Retrieve the construction site
+            let construction_site = self.app_server_mut().crypto_site.borrow_mut();
+
+            // Retrieve the builder
+            use rosenpass_util::build::ConstructionSite as C;
+            let maybe_builder = match construction_site {
+                C::Builder(builder) => Some(builder),
+                C::Product(_) => None,
+                C::Void => {
+                    return Err(anyhow::Error::msg("CryptoServer construction side is void"))
+                        .einternal();
+                }
+            };
+
+            // Retrieve a reference to the keypair
+            let Some(BuildCryptoServer {
+                ref mut keypair, ..
+            }) = maybe_builder
+            else {
+                return Err(anyhow::Error::msg("CryptoServer already built")).ealready_supplied();
+            };
+
+            // Supply the keypair to the CryptoServer
+            keypair
+                .insert(crate::protocol::Keypair { sk, pk })
+                .discard_result();
+
+            // Actually construct the CryptoServer
+            construction_site
+                .erect()
+                .map_err(|e| anyhow::Error::msg(format!("Error erecting the CryptoServer {e:?}")))
+                .einternal()?;
+
+            Ok(())
+        });
+
+        // Handle errors
+        use supply_keypair_response_status as status;
+        let status = match outcome {
+            Ok(()) => status::OK,
+            Err(e) => {
+                let lvl = match e.status {
+                    status::INTERNAL_ERROR => log::Level::Warn,
+                    _ => log::Level::Debug,
+                };
+
+                log::log!(
+                    lvl,
+                    "Error while processing API Request.\n    Request: {:?}\n    Error: {:?}",
+                    req,
+                    e.cause
+                );
+
+                if e.status == status::INTERNAL_ERROR {
+                    return Err(e.cause);
+                }
+
+                e.status
+            }
+        };
+
+        res.payload.status = status;
+
+        Ok(())
+    }
+
+    fn add_listen_socket(
+        &mut self,
+        _req: &super::boilerplate::AddListenSocketRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::boilerplate::AddListenSocketResponse,
+    ) -> anyhow::Result<()> {
+        // Retrieve file descriptor
+        let sock_res = run(|| -> anyhow::Result<mio::net::UdpSocket> {
+            let sock = req_fds
+                .pop_front()
+                .context("Invalid request – socket missing.")?;
+            // TODO: We need to have this outside linux
+            #[cfg(target_os = "linux")]
+            rosenpass_util::fd::GetSocketProtocol::demand_udp_socket(&sock)?;
+            let sock = std::net::UdpSocket::from(sock);
+            sock.set_nonblocking(true)?;
+            mio::net::UdpSocket::from_std(sock).ok()
+        });
+
+        let sock = match sock_res {
+            Ok(sock) => sock,
+            Err(e) => {
+                log::debug!("Error processing AddListenSocket API request: {e:?}");
+                res.payload.status = add_listen_socket_response_status::INVALID_REQUEST;
+                return Ok(());
+            }
+        };
+
+        // Register socket
+        let reg_result = self.app_server_mut().register_listen_socket(sock);
+
+        if let Err(internal_error) = reg_result {
+            log::warn!("Internal error processing AddListenSocket API request: {internal_error:?}");
+            res.payload.status = add_listen_socket_response_status::INTERNAL_ERROR;
+            return Ok(());
+        };
+
+        res.payload.status = add_listen_socket_response_status::OK;
+        Ok(())
+    }
+
+    fn add_psk_broker(
+        &mut self,
+        _req: &super::boilerplate::AddPskBrokerRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::boilerplate::AddPskBrokerResponse,
+    ) -> anyhow::Result<()> {
+        // Retrieve file descriptor
+        let sock_res = run(|| {
+            let sock = req_fds
+                .pop_front()
+                .context("Invalid request – socket missing.")?;
+            mio::net::UnixStream::from_fd(sock)
+        });
+
+        // Handle errors
+        let sock = match sock_res {
+            Ok(sock) => sock,
+            Err(e) => {
+                log::debug!(
+                    "Request found to be invalid while processing AddPskBroker API request: {e:?}"
+                );
+                res.payload.status = add_psk_broker_response_status::INVALID_REQUEST;
+                return Ok(());
+            }
+        };
+
+        // Register Socket
+        let client = Box::new(MioBrokerClient::new(sock));
+
+        // Workaround: The broker code is currently impressively overcomplicated. Brokers are
+        // stored in a hash map but the hash map key used is just a counter so a vector could
+        // have been used. Broker configuration is abstracted, different peers can have different
+        // brokers but there is no facility to add multiple brokers in practice. The broker index
+        // uses a `Public` wrapper without actually holding any cryptographic data. Even the broker
+        // configuration uses a trait abstraction for no discernible reason and a lot of the code
+        // introduces pointless, single-field wrapper structs.
+        // We should use an implement-what-is-actually-needed strategy next time.
+        // The Broker code needs to be slimmed down, the right direction to go is probably to
+        // just add event and capability support to the API and use the API to deliver OSK events.
+        //
+        // For now, we just replace the latest broker.
+        let erase_ptr = {
+            use crate::app_server::BrokerStorePtr;
+            //
+            use rosenpass_secret_memory::Public;
+            use zerocopy::AsBytes;
+            (self.app_server().brokers.store.len() - 1)
+                .apply(|x| x as u64)
+                .apply(|x| Public::from_slice(x.as_bytes()))
+                .apply(BrokerStorePtr)
+        };
+
+        let register_result = run(|| {
+            let srv = self.app_server_mut();
+            srv.unregister_broker(erase_ptr)?;
+            srv.register_broker(client)
+        });
+
+        if let Err(e) = register_result {
+            log::warn!("Internal error while processing AddPskBroker API request: {e:?}");
+            res.payload.status = add_psk_broker_response_status::INTERNAL_ERROR;
+            return Ok(());
+        }
+
+        res.payload.status = add_psk_broker_response_status::OK;
+        Ok(())
+    }
+}

rosenpass/src/api/boilerplate/byte_slice_ext.rs

@@ -0,0 +1,222 @@
+use zerocopy::{ByteSlice, Ref};
+
+use rosenpass_util::zerocopy::{RefMaker, ZerocopySliceExt};
+
+use super::{
+    PingRequest, PingResponse, RawMsgType, RefMakerRawMsgTypeExt, RequestMsgType, RequestRef,
+    ResponseMsgType, ResponseRef, SupplyKeypairRequest, SupplyKeypairResponse,
+};
+
+pub trait ByteSliceRefExt: ByteSlice {
+    fn msg_type_maker(self) -> RefMaker<Self, RawMsgType> {
+        self.zk_ref_maker()
+    }
+
+    fn msg_type(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse()
+    }
+
+    fn msg_type_from_prefix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn msg_type_from_suffix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn request_msg_type(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker().parse_request_msg_type()
+    }
+
+    fn request_msg_type_from_prefix(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker()
+            .from_prefix()?
+            .parse_request_msg_type()
+    }
+
+    fn request_msg_type_from_suffix(self) -> anyhow::Result<RequestMsgType> {
+        self.msg_type_maker()
+            .from_suffix()?
+            .parse_request_msg_type()
+    }
+
+    fn response_msg_type(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker().parse_response_msg_type()
+    }
+
+    fn response_msg_type_from_prefix(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker()
+            .from_prefix()?
+            .parse_response_msg_type()
+    }
+
+    fn response_msg_type_from_suffix(self) -> anyhow::Result<ResponseMsgType> {
+        self.msg_type_maker()
+            .from_suffix()?
+            .parse_response_msg_type()
+    }
+
+    fn parse_request(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse(self)
+    }
+
+    fn parse_request_from_prefix(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse_from_prefix(self)
+    }
+
+    fn parse_request_from_suffix(self) -> anyhow::Result<RequestRef<Self>> {
+        RequestRef::parse_from_suffix(self)
+    }
+
+    fn parse_response(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse(self)
+    }
+
+    fn parse_response_from_prefix(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse_from_prefix(self)
+    }
+
+    fn parse_response_from_suffix(self) -> anyhow::Result<ResponseRef<Self>> {
+        ResponseRef::parse_from_suffix(self)
+    }
+
+    fn ping_request_maker(self) -> RefMaker<Self, PingRequest> {
+        self.zk_ref_maker()
+    }
+
+    fn ping_request(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse()
+    }
+
+    fn ping_request_from_prefix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn ping_request_from_suffix(self) -> anyhow::Result<Ref<Self, PingRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn ping_response_maker(self) -> RefMaker<Self, PingResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn ping_response(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse()
+    }
+
+    fn ping_response_from_prefix(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn ping_response_from_suffix(self) -> anyhow::Result<Ref<Self, PingResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn supply_keypair_request(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse()
+    }
+
+    fn supply_keypair_request_from_prefix(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn supply_keypair_request_from_suffix(self) -> anyhow::Result<Ref<Self, SupplyKeypairRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn supply_keypair_response_maker(self) -> RefMaker<Self, SupplyKeypairResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn supply_keypair_response(self) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse()
+    }
+
+    fn supply_keypair_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn supply_keypair_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, SupplyKeypairResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_listen_socket_request(self) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse()
+    }
+
+    fn add_listen_socket_request_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_listen_socket_request_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_listen_socket_response_maker(self) -> RefMaker<Self, super::AddListenSocketResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn add_listen_socket_response(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse()
+    }
+
+    fn add_listen_socket_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_listen_socket_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddListenSocketResponse>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_psk_broker_request(self) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse()
+    }
+
+    fn add_psk_broker_request_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_psk_broker_request_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerRequest>> {
+        self.zk_parse_suffix()
+    }
+
+    fn add_psk_broker_response_maker(self) -> RefMaker<Self, super::AddPskBrokerResponse> {
+        self.zk_ref_maker()
+    }
+
+    fn add_psk_broker_response(self) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse()
+    }
+
+    fn add_psk_broker_response_from_prefix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse_prefix()
+    }
+
+    fn add_psk_broker_response_from_suffix(
+        self,
+    ) -> anyhow::Result<Ref<Self, super::AddPskBrokerResponse>> {
+        self.zk_parse_suffix()
+    }
+}
+
+impl<B: ByteSlice> ByteSliceRefExt for B {}

rosenpass/src/api/boilerplate/message_trait.rs

@@ -0,0 +1,29 @@
+use zerocopy::{ByteSliceMut, Ref};
+
+use rosenpass_util::zerocopy::RefMaker;
+
+use super::RawMsgType;
+
+pub trait Message {
+    type Payload;
+    type MessageClass: Into<RawMsgType>;
+    const MESSAGE_TYPE: Self::MessageClass;
+
+    fn from_payload(payload: Self::Payload) -> Self;
+    fn init(&mut self);
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>>;
+}
+
+pub trait ZerocopyResponseMakerSetupMessageExt<B, T> {
+    fn setup_msg(self) -> anyhow::Result<Ref<B, T>>;
+}
+
+impl<B, T> ZerocopyResponseMakerSetupMessageExt<B, T> for RefMaker<B, T>
+where
+    B: ByteSliceMut,
+    T: Message,
+{
+    fn setup_msg(self) -> anyhow::Result<Ref<B, T>> {
+        T::setup(self.into_buf())
+    }
+}

rosenpass/src/api/boilerplate/message_type.rs

@@ -0,0 +1,162 @@
+use hex_literal::hex;
+use rosenpass_util::zerocopy::RefMaker;
+use zerocopy::ByteSlice;
+
+use crate::RosenpassError::{self, InvalidApiMessageType};
+
+pub type RawMsgType = u128;
+
+// constants generated by gen-ipc-msg-types:
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Ping Request
+pub const PING_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("2397 3ecc c441 704d    0b02 ea31 45d3 4999"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Ping Response
+pub const PING_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("4ec7 f6f0 2bbc ba64    48f1 da14 c7cf 0260"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Supply Keypair Request
+const SUPPLY_KEYPAIR_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("ac91 a5a6 4f4b 21d0    ac7f 9b55 74f7 3529"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Supply Keypair Response
+const SUPPLY_KEYPAIR_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("f2dc 49bd e261 5f10    40b7 3c16 ec61 edb9"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Listen Socket Request
+const ADD_LISTEN_SOCKET_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("3f21 434f 87cc a08c    02c4 61e4 0816 c7da"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Listen Socket Response
+const ADD_LISTEN_SOCKET_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("45d5 0f0d 93f0 6105    98f2 9469 5dfd 5f36"));
+
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Psk Broker Request
+const ADD_PSK_BROKER_REQUEST: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("d798 b8dc bd61 5cab    8df1 c63d e4eb a2d1"));
+// hash domain hash of: Rosenpass IPC API -> Rosenpass Protocol Server -> Add Psk Broker Response
+const ADD_PSK_BROKER_RESPONSE: RawMsgType =
+    RawMsgType::from_le_bytes(hex!("bd25 e418 ffb0 6930    248b 217e 2fae e353"));
+
+pub trait MessageAttributes {
+    fn message_size(&self) -> usize;
+}
+
+#[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+pub enum RequestMsgType {
+    Ping,
+    SupplyKeypair,
+    AddListenSocket,
+    AddPskBroker,
+}
+
+#[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+pub enum ResponseMsgType {
+    Ping,
+    SupplyKeypair,
+    AddListenSocket,
+    AddPskBroker,
+}
+
+impl MessageAttributes for RequestMsgType {
+    fn message_size(&self) -> usize {
+        match self {
+            Self::Ping => std::mem::size_of::<super::PingRequest>(),
+            Self::SupplyKeypair => std::mem::size_of::<super::SupplyKeypairRequest>(),
+            Self::AddListenSocket => std::mem::size_of::<super::AddListenSocketRequest>(),
+            Self::AddPskBroker => std::mem::size_of::<super::AddPskBrokerRequest>(),
+        }
+    }
+}
+
+impl MessageAttributes for ResponseMsgType {
+    fn message_size(&self) -> usize {
+        match self {
+            Self::Ping => std::mem::size_of::<super::PingResponse>(),
+            Self::SupplyKeypair => std::mem::size_of::<super::SupplyKeypairResponse>(),
+            Self::AddListenSocket => std::mem::size_of::<super::AddListenSocketResponse>(),
+            Self::AddPskBroker => std::mem::size_of::<super::AddPskBrokerResponse>(),
+        }
+    }
+}
+
+impl TryFrom<RawMsgType> for RequestMsgType {
+    type Error = RosenpassError;
+
+    fn try_from(value: RawMsgType) -> Result<Self, Self::Error> {
+        use RequestMsgType as E;
+        Ok(match value {
+            self::PING_REQUEST => E::Ping,
+            self::SUPPLY_KEYPAIR_REQUEST => E::SupplyKeypair,
+            self::ADD_LISTEN_SOCKET_REQUEST => E::AddListenSocket,
+            self::ADD_PSK_BROKER_REQUEST => E::AddPskBroker,
+            _ => return Err(InvalidApiMessageType(value)),
+        })
+    }
+}
+
+impl From<RequestMsgType> for RawMsgType {
+    fn from(val: RequestMsgType) -> Self {
+        use RequestMsgType as E;
+        match val {
+            E::Ping => self::PING_REQUEST,
+            E::SupplyKeypair => self::SUPPLY_KEYPAIR_REQUEST,
+            E::AddListenSocket => self::ADD_LISTEN_SOCKET_REQUEST,
+            E::AddPskBroker => self::ADD_PSK_BROKER_REQUEST,
+        }
+    }
+}
+
+impl TryFrom<RawMsgType> for ResponseMsgType {
+    type Error = RosenpassError;
+
+    fn try_from(value: RawMsgType) -> Result<Self, Self::Error> {
+        use ResponseMsgType as E;
+        Ok(match value {
+            self::PING_RESPONSE => E::Ping,
+            self::SUPPLY_KEYPAIR_RESPONSE => E::SupplyKeypair,
+            self::ADD_LISTEN_SOCKET_RESPONSE => E::AddListenSocket,
+            self::ADD_PSK_BROKER_RESPONSE => E::AddPskBroker,
+            _ => return Err(InvalidApiMessageType(value)),
+        })
+    }
+}
+
+impl From<ResponseMsgType> for RawMsgType {
+    fn from(val: ResponseMsgType) -> Self {
+        use ResponseMsgType as E;
+        match val {
+            E::Ping => self::PING_RESPONSE,
+            E::SupplyKeypair => self::SUPPLY_KEYPAIR_RESPONSE,
+            E::AddListenSocket => self::ADD_LISTEN_SOCKET_RESPONSE,
+            E::AddPskBroker => self::ADD_PSK_BROKER_RESPONSE,
+        }
+    }
+}
+
+pub trait RawMsgTypeExt {
+    fn into_request_msg_type(self) -> Result<RequestMsgType, RosenpassError>;
+    fn into_response_msg_type(self) -> Result<ResponseMsgType, RosenpassError>;
+}
+
+impl RawMsgTypeExt for RawMsgType {
+    fn into_request_msg_type(self) -> Result<RequestMsgType, RosenpassError> {
+        self.try_into()
+    }
+
+    fn into_response_msg_type(self) -> Result<ResponseMsgType, RosenpassError> {
+        self.try_into()
+    }
+}
+
+pub trait RefMakerRawMsgTypeExt {
+    fn parse_request_msg_type(self) -> anyhow::Result<RequestMsgType>;
+    fn parse_response_msg_type(self) -> anyhow::Result<ResponseMsgType>;
+}
+
+impl<B: ByteSlice> RefMakerRawMsgTypeExt for RefMaker<B, RawMsgType> {
+    fn parse_request_msg_type(self) -> anyhow::Result<RequestMsgType> {
+        Ok(self.parse()?.read().try_into()?)
+    }
+
+    fn parse_response_msg_type(self) -> anyhow::Result<ResponseMsgType> {
+        Ok(self.parse()?.read().try_into()?)
+    }
+}

rosenpass/src/api/boilerplate/mod.rs

@@ -0,0 +1,17 @@
+mod byte_slice_ext;
+mod message_trait;
+mod message_type;
+mod payload;
+mod request_ref;
+mod request_response;
+mod response_ref;
+mod server;
+
+pub use byte_slice_ext::*;
+pub use message_trait::*;
+pub use message_type::*;
+pub use payload::*;
+pub use request_ref::*;
+pub use request_response::*;
+pub use response_ref::*;
+pub use server::*;

rosenpass/src/api/boilerplate/payload.rs

@@ -0,0 +1,351 @@
+use rosenpass_util::zerocopy::ZerocopyMutSliceExt;
+use zerocopy::{AsBytes, ByteSliceMut, FromBytes, FromZeroes, Ref};
+
+use super::{Message, RawMsgType, RequestMsgType, ResponseMsgType};
+
+/// Size required to fit any message in binary form
+pub const MAX_REQUEST_LEN: usize = 2500; // TODO fix this
+pub const MAX_RESPONSE_LEN: usize = 2500; // TODO fix this
+pub const MAX_REQUEST_FDS: usize = 2;
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct Envelope<M: AsBytes + FromBytes> {
+    /// Which message this is
+    pub msg_type: RawMsgType,
+    /// The actual Paylod
+    pub payload: M,
+}
+
+pub type RequestEnvelope<M> = Envelope<M>;
+pub type ResponseEnvelope<M> = Envelope<M>;
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct PingRequestPayload {
+    /// Randomly generated connection id
+    pub echo: [u8; 256],
+}
+
+pub type PingRequest = RequestEnvelope<PingRequestPayload>;
+
+impl PingRequest {
+    pub fn new(echo: [u8; 256]) -> Self {
+        Self::from_payload(PingRequestPayload { echo })
+    }
+}
+
+impl Message for PingRequest {
+    type Payload = PingRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::Ping;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct PingResponsePayload {
+    /// Randomly generated connection id
+    pub echo: [u8; 256],
+}
+
+pub type PingResponse = ResponseEnvelope<PingResponsePayload>;
+
+impl PingResponse {
+    pub fn new(echo: [u8; 256]) -> Self {
+        Self::from_payload(PingResponsePayload { echo })
+    }
+}
+
+impl Message for PingResponse {
+    type Payload = PingResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::Ping;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct SupplyKeypairRequestPayload {}
+
+pub type SupplyKeypairRequest = RequestEnvelope<SupplyKeypairRequestPayload>;
+
+impl Default for SupplyKeypairRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SupplyKeypairRequest {
+    pub fn new() -> Self {
+        Self::from_payload(SupplyKeypairRequestPayload {})
+    }
+}
+
+impl Message for SupplyKeypairRequest {
+    type Payload = SupplyKeypairRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::SupplyKeypair;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod supply_keypair_response_status {
+    pub const OK: u128 = 0;
+    pub const KEYPAIR_ALREADY_SUPPLIED: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+    pub const INVALID_REQUEST: u128 = 3;
+    pub const IO_ERROR: u128 = 4;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct SupplyKeypairResponsePayload {
+    pub status: u128,
+}
+
+pub type SupplyKeypairResponse = ResponseEnvelope<SupplyKeypairResponsePayload>;
+
+impl SupplyKeypairResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(SupplyKeypairResponsePayload { status })
+    }
+}
+
+impl Message for SupplyKeypairResponse {
+    type Payload = SupplyKeypairResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::SupplyKeypair;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddListenSocketRequestPayload {}
+
+pub type AddListenSocketRequest = RequestEnvelope<AddListenSocketRequestPayload>;
+
+impl Default for AddListenSocketRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AddListenSocketRequest {
+    pub fn new() -> Self {
+        Self::from_payload(AddListenSocketRequestPayload {})
+    }
+}
+
+impl Message for AddListenSocketRequest {
+    type Payload = AddListenSocketRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::AddListenSocket;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod add_listen_socket_response_status {
+    pub const OK: u128 = 0;
+    pub const INVALID_REQUEST: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddListenSocketResponsePayload {
+    pub status: u128,
+}
+
+pub type AddListenSocketResponse = ResponseEnvelope<AddListenSocketResponsePayload>;
+
+impl AddListenSocketResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(AddListenSocketResponsePayload { status })
+    }
+}
+
+impl Message for AddListenSocketResponse {
+    type Payload = AddListenSocketResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::AddListenSocket;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddPskBrokerRequestPayload {}
+
+pub type AddPskBrokerRequest = RequestEnvelope<AddPskBrokerRequestPayload>;
+
+impl Default for AddPskBrokerRequest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AddPskBrokerRequest {
+    pub fn new() -> Self {
+        Self::from_payload(AddPskBrokerRequestPayload {})
+    }
+}
+
+impl Message for AddPskBrokerRequest {
+    type Payload = AddPskBrokerRequestPayload;
+    type MessageClass = RequestMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = RequestMsgType::AddPskBroker;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}
+
+pub mod add_psk_broker_response_status {
+    pub const OK: u128 = 0;
+    pub const INVALID_REQUEST: u128 = 1;
+    pub const INTERNAL_ERROR: u128 = 2;
+}
+
+#[repr(packed)]
+#[derive(Debug, Copy, Clone, Hash, AsBytes, FromBytes, FromZeroes, PartialEq, Eq)]
+pub struct AddPskBrokerResponsePayload {
+    pub status: u128,
+}
+
+pub type AddPskBrokerResponse = ResponseEnvelope<AddPskBrokerResponsePayload>;
+
+impl AddPskBrokerResponse {
+    pub fn new(status: u128) -> Self {
+        Self::from_payload(AddPskBrokerResponsePayload { status })
+    }
+}
+
+impl Message for AddPskBrokerResponse {
+    type Payload = AddPskBrokerResponsePayload;
+    type MessageClass = ResponseMsgType;
+    const MESSAGE_TYPE: Self::MessageClass = ResponseMsgType::AddPskBroker;
+
+    fn from_payload(payload: Self::Payload) -> Self {
+        Self {
+            msg_type: Self::MESSAGE_TYPE.into(),
+            payload,
+        }
+    }
+
+    fn setup<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self>> {
+        let mut r: Ref<B, Self> = buf.zk_zeroized()?;
+        r.init();
+        Ok(r)
+    }
+
+    fn init(&mut self) {
+        self.msg_type = Self::MESSAGE_TYPE.into();
+    }
+}

rosenpass/src/api/boilerplate/request_ref.rs

@@ -0,0 +1,146 @@
+use anyhow::ensure;
+
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{ByteSliceRefExt, MessageAttributes, PingRequest, RequestMsgType};
+
+struct RequestRefMaker<B> {
+    buf: B,
+    msg_type: RequestMsgType,
+}
+
+impl<B: ByteSlice> RequestRef<B> {
+    pub fn parse(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.parse()
+    }
+
+    pub fn parse_from_prefix(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.from_prefix()?.parse()
+    }
+
+    pub fn parse_from_suffix(buf: B) -> anyhow::Result<Self> {
+        RequestRefMaker::new(buf)?.from_suffix()?.parse()
+    }
+
+    pub fn message_type(&self) -> RequestMsgType {
+        match self {
+            Self::Ping(_) => RequestMsgType::Ping,
+            Self::SupplyKeypair(_) => RequestMsgType::SupplyKeypair,
+            Self::AddListenSocket(_) => RequestMsgType::AddListenSocket,
+            Self::AddPskBroker(_) => RequestMsgType::AddPskBroker,
+        }
+    }
+}
+
+impl<B> From<Ref<B, PingRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, PingRequest>) -> Self {
+        Self::Ping(v)
+    }
+}
+
+impl<B> From<Ref<B, super::SupplyKeypairRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::SupplyKeypairRequest>) -> Self {
+        Self::SupplyKeypair(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddListenSocketRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::AddListenSocketRequest>) -> Self {
+        Self::AddListenSocket(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddPskBrokerRequest>> for RequestRef<B> {
+    fn from(v: Ref<B, super::AddPskBrokerRequest>) -> Self {
+        Self::AddPskBroker(v)
+    }
+}
+
+impl<B: ByteSlice> RequestRefMaker<B> {
+    fn new(buf: B) -> anyhow::Result<Self> {
+        let msg_type = buf.deref().request_msg_type_from_prefix()?;
+        Ok(Self { buf, msg_type })
+    }
+
+    fn target_size(&self) -> usize {
+        self.msg_type.message_size()
+    }
+
+    fn parse(self) -> anyhow::Result<RequestRef<B>> {
+        Ok(match self.msg_type {
+            RequestMsgType::Ping => RequestRef::Ping(self.buf.ping_request()?),
+            RequestMsgType::SupplyKeypair => {
+                RequestRef::SupplyKeypair(self.buf.supply_keypair_request()?)
+            }
+            RequestMsgType::AddListenSocket => {
+                RequestRef::AddListenSocket(self.buf.add_listen_socket_request()?)
+            }
+            RequestMsgType::AddPskBroker => {
+                RequestRef::AddPskBroker(self.buf.add_psk_broker_request()?)
+            }
+        })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_prefix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_suffix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.buf.len() - self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    pub fn ensure_fit(&self) -> anyhow::Result<()> {
+        let have = self.buf.len();
+        let need = self.target_size();
+        ensure!(
+            need <= have,
+            "Buffer is undersized at {have} bytes (need {need} bytes)!"
+        );
+        Ok(())
+    }
+}
+
+pub enum RequestRef<B> {
+    Ping(Ref<B, PingRequest>),
+    SupplyKeypair(Ref<B, super::SupplyKeypairRequest>),
+    AddListenSocket(Ref<B, super::AddListenSocketRequest>),
+    AddPskBroker(Ref<B, super::AddPskBrokerRequest>),
+}
+
+impl<B> RequestRef<B>
+where
+    B: ByteSlice,
+{
+    pub fn bytes(&self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes(),
+            Self::SupplyKeypair(r) => r.bytes(),
+            Self::AddListenSocket(r) => r.bytes(),
+            Self::AddPskBroker(r) => r.bytes(),
+        }
+    }
+}
+
+impl<B> RequestRef<B>
+where
+    B: ByteSliceMut,
+{
+    pub fn bytes_mut(&mut self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes_mut(),
+            Self::SupplyKeypair(r) => r.bytes_mut(),
+            Self::AddListenSocket(r) => r.bytes_mut(),
+            Self::AddPskBroker(r) => r.bytes_mut(),
+        }
+    }
+}

rosenpass/src/api/boilerplate/request_response.rs

@@ -0,0 +1,190 @@
+use rosenpass_util::zerocopy::{
+    RefMaker, ZerocopyEmancipateExt, ZerocopyEmancipateMutExt, ZerocopySliceExt,
+};
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{Message, PingRequest, PingResponse};
+use super::{RequestRef, ResponseRef, ZerocopyResponseMakerSetupMessageExt};
+
+pub trait RequestMsg: Sized + Message {
+    type ResponseMsg: ResponseMsg;
+
+    fn zk_response_maker<B: ByteSlice>(buf: B) -> RefMaker<B, Self::ResponseMsg> {
+        buf.zk_ref_maker()
+    }
+
+    fn setup_response<B: ByteSliceMut>(buf: B) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).setup_msg()
+    }
+
+    fn setup_response_from_prefix<B: ByteSliceMut>(
+        buf: B,
+    ) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).from_prefix()?.setup_msg()
+    }
+
+    fn setup_response_from_suffix<B: ByteSliceMut>(
+        buf: B,
+    ) -> anyhow::Result<Ref<B, Self::ResponseMsg>> {
+        Self::zk_response_maker(buf).from_prefix()?.setup_msg()
+    }
+}
+
+pub trait ResponseMsg: Message {
+    type RequestMsg: RequestMsg;
+}
+
+impl RequestMsg for PingRequest {
+    type ResponseMsg = PingResponse;
+}
+
+impl ResponseMsg for PingResponse {
+    type RequestMsg = PingRequest;
+}
+
+impl RequestMsg for super::SupplyKeypairRequest {
+    type ResponseMsg = super::SupplyKeypairResponse;
+}
+
+impl ResponseMsg for super::SupplyKeypairResponse {
+    type RequestMsg = super::SupplyKeypairRequest;
+}
+
+impl RequestMsg for super::AddListenSocketRequest {
+    type ResponseMsg = super::AddListenSocketResponse;
+}
+
+impl ResponseMsg for super::AddListenSocketResponse {
+    type RequestMsg = super::AddListenSocketRequest;
+}
+
+impl RequestMsg for super::AddPskBrokerRequest {
+    type ResponseMsg = super::AddPskBrokerResponse;
+}
+
+impl ResponseMsg for super::AddPskBrokerResponse {
+    type RequestMsg = super::AddPskBrokerRequest;
+}
+
+pub type PingPair<B1, B2> = (Ref<B1, PingRequest>, Ref<B2, PingResponse>);
+pub type SupplyKeypairPair<B1, B2> = (
+    Ref<B1, super::SupplyKeypairRequest>,
+    Ref<B2, super::SupplyKeypairResponse>,
+);
+pub type AddListenSocketPair<B1, B2> = (
+    Ref<B1, super::AddListenSocketRequest>,
+    Ref<B2, super::AddListenSocketResponse>,
+);
+pub type AddPskBrokerPair<B1, B2> = (
+    Ref<B1, super::AddPskBrokerRequest>,
+    Ref<B2, super::AddPskBrokerResponse>,
+);
+
+pub enum RequestResponsePair<B1, B2> {
+    Ping(PingPair<B1, B2>),
+    SupplyKeypair(SupplyKeypairPair<B1, B2>),
+    AddListenSocket(AddListenSocketPair<B1, B2>),
+    AddPskBroker(AddPskBrokerPair<B1, B2>),
+}
+
+impl<B1, B2> From<PingPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: PingPair<B1, B2>) -> Self {
+        RequestResponsePair::Ping(v)
+    }
+}
+
+impl<B1, B2> From<SupplyKeypairPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: SupplyKeypairPair<B1, B2>) -> Self {
+        RequestResponsePair::SupplyKeypair(v)
+    }
+}
+
+impl<B1, B2> From<AddListenSocketPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: AddListenSocketPair<B1, B2>) -> Self {
+        RequestResponsePair::AddListenSocket(v)
+    }
+}
+
+impl<B1, B2> From<AddPskBrokerPair<B1, B2>> for RequestResponsePair<B1, B2> {
+    fn from(v: AddPskBrokerPair<B1, B2>) -> Self {
+        RequestResponsePair::AddPskBroker(v)
+    }
+}
+
+impl<B1, B2> RequestResponsePair<B1, B2>
+where
+    B1: ByteSlice,
+    B2: ByteSlice,
+{
+    pub fn both(&self) -> (RequestRef<&[u8]>, ResponseRef<&[u8]>) {
+        match self {
+            Self::Ping((req, res)) => {
+                let req = RequestRef::Ping(req.emancipate());
+                let res = ResponseRef::Ping(res.emancipate());
+                (req, res)
+            }
+            Self::SupplyKeypair((req, res)) => {
+                let req = RequestRef::SupplyKeypair(req.emancipate());
+                let res = ResponseRef::SupplyKeypair(res.emancipate());
+                (req, res)
+            }
+            Self::AddListenSocket((req, res)) => {
+                let req = RequestRef::AddListenSocket(req.emancipate());
+                let res = ResponseRef::AddListenSocket(res.emancipate());
+                (req, res)
+            }
+            Self::AddPskBroker((req, res)) => {
+                let req = RequestRef::AddPskBroker(req.emancipate());
+                let res = ResponseRef::AddPskBroker(res.emancipate());
+                (req, res)
+            }
+        }
+    }
+
+    pub fn request(&self) -> RequestRef<&[u8]> {
+        self.both().0
+    }
+
+    pub fn response(&self) -> ResponseRef<&[u8]> {
+        self.both().1
+    }
+}
+
+impl<B1, B2> RequestResponsePair<B1, B2>
+where
+    B1: ByteSliceMut,
+    B2: ByteSliceMut,
+{
+    pub fn both_mut(&mut self) -> (RequestRef<&mut [u8]>, ResponseRef<&mut [u8]>) {
+        match self {
+            Self::Ping((req, res)) => {
+                let req = RequestRef::Ping(req.emancipate_mut());
+                let res = ResponseRef::Ping(res.emancipate_mut());
+                (req, res)
+            }
+            Self::SupplyKeypair((req, res)) => {
+                let req = RequestRef::SupplyKeypair(req.emancipate_mut());
+                let res = ResponseRef::SupplyKeypair(res.emancipate_mut());
+                (req, res)
+            }
+            Self::AddListenSocket((req, res)) => {
+                let req = RequestRef::AddListenSocket(req.emancipate_mut());
+                let res = ResponseRef::AddListenSocket(res.emancipate_mut());
+                (req, res)
+            }
+            Self::AddPskBroker((req, res)) => {
+                let req = RequestRef::AddPskBroker(req.emancipate_mut());
+                let res = ResponseRef::AddPskBroker(res.emancipate_mut());
+                (req, res)
+            }
+        }
+    }
+
+    pub fn request_mut(&mut self) -> RequestRef<&mut [u8]> {
+        self.both_mut().0
+    }
+
+    pub fn response_mut(&mut self) -> ResponseRef<&mut [u8]> {
+        self.both_mut().1
+    }
+}

rosenpass/src/api/boilerplate/response_ref.rs

@@ -0,0 +1,147 @@
+// TODO: This is copied verbatim from ResponseRef…not pretty
+use anyhow::ensure;
+
+use zerocopy::{ByteSlice, ByteSliceMut, Ref};
+
+use super::{ByteSliceRefExt, MessageAttributes, PingResponse, ResponseMsgType};
+
+struct ResponseRefMaker<B> {
+    buf: B,
+    msg_type: ResponseMsgType,
+}
+
+impl<B: ByteSlice> ResponseRef<B> {
+    pub fn parse(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.parse()
+    }
+
+    pub fn parse_from_prefix(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.from_prefix()?.parse()
+    }
+
+    pub fn parse_from_suffix(buf: B) -> anyhow::Result<Self> {
+        ResponseRefMaker::new(buf)?.from_suffix()?.parse()
+    }
+
+    pub fn message_type(&self) -> ResponseMsgType {
+        match self {
+            Self::Ping(_) => ResponseMsgType::Ping,
+            Self::SupplyKeypair(_) => ResponseMsgType::SupplyKeypair,
+            Self::AddListenSocket(_) => ResponseMsgType::AddListenSocket,
+            Self::AddPskBroker(_) => ResponseMsgType::AddPskBroker,
+        }
+    }
+}
+
+impl<B> From<Ref<B, PingResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, PingResponse>) -> Self {
+        Self::Ping(v)
+    }
+}
+
+impl<B> From<Ref<B, super::SupplyKeypairResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::SupplyKeypairResponse>) -> Self {
+        Self::SupplyKeypair(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddListenSocketResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::AddListenSocketResponse>) -> Self {
+        Self::AddListenSocket(v)
+    }
+}
+
+impl<B> From<Ref<B, super::AddPskBrokerResponse>> for ResponseRef<B> {
+    fn from(v: Ref<B, super::AddPskBrokerResponse>) -> Self {
+        Self::AddPskBroker(v)
+    }
+}
+
+impl<B: ByteSlice> ResponseRefMaker<B> {
+    fn new(buf: B) -> anyhow::Result<Self> {
+        let msg_type = buf.deref().response_msg_type_from_prefix()?;
+        Ok(Self { buf, msg_type })
+    }
+
+    fn target_size(&self) -> usize {
+        self.msg_type.message_size()
+    }
+
+    fn parse(self) -> anyhow::Result<ResponseRef<B>> {
+        Ok(match self.msg_type {
+            ResponseMsgType::Ping => ResponseRef::Ping(self.buf.ping_response()?),
+            ResponseMsgType::SupplyKeypair => {
+                ResponseRef::SupplyKeypair(self.buf.supply_keypair_response()?)
+            }
+            ResponseMsgType::AddListenSocket => {
+                ResponseRef::AddListenSocket(self.buf.add_listen_socket_response()?)
+            }
+            ResponseMsgType::AddPskBroker => {
+                ResponseRef::AddPskBroker(self.buf.add_psk_broker_response()?)
+            }
+        })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_prefix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    #[allow(clippy::wrong_self_convention)]
+    fn from_suffix(self) -> anyhow::Result<Self> {
+        self.ensure_fit()?;
+        let point = self.buf.len() - self.target_size();
+        let Self { buf, msg_type } = self;
+        let (buf, _) = buf.split_at(point);
+        Ok(Self { buf, msg_type })
+    }
+
+    pub fn ensure_fit(&self) -> anyhow::Result<()> {
+        let have = self.buf.len();
+        let need = self.target_size();
+        ensure!(
+            need <= have,
+            "Buffer is undersized at {have} bytes (need {need} bytes)!"
+        );
+        Ok(())
+    }
+}
+
+pub enum ResponseRef<B> {
+    Ping(Ref<B, PingResponse>),
+    SupplyKeypair(Ref<B, super::SupplyKeypairResponse>),
+    AddListenSocket(Ref<B, super::AddListenSocketResponse>),
+    AddPskBroker(Ref<B, super::AddPskBrokerResponse>),
+}
+
+impl<B> ResponseRef<B>
+where
+    B: ByteSlice,
+{
+    pub fn bytes(&self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes(),
+            Self::SupplyKeypair(r) => r.bytes(),
+            Self::AddListenSocket(r) => r.bytes(),
+            Self::AddPskBroker(r) => r.bytes(),
+        }
+    }
+}
+
+impl<B> ResponseRef<B>
+where
+    B: ByteSliceMut,
+{
+    pub fn bytes_mut(&mut self) -> &[u8] {
+        match self {
+            Self::Ping(r) => r.bytes_mut(),
+            Self::SupplyKeypair(r) => r.bytes_mut(),
+            Self::AddListenSocket(r) => r.bytes_mut(),
+            Self::AddPskBroker(r) => r.bytes_mut(),
+        }
+    }
+}

rosenpass/src/api/boilerplate/server.rs

@@ -0,0 +1,94 @@
+use super::{ByteSliceRefExt, Message, PingRequest, PingResponse, RequestRef, RequestResponsePair};
+use std::{collections::VecDeque, os::fd::OwnedFd};
+use zerocopy::{ByteSlice, ByteSliceMut};
+
+pub trait Server {
+    fn ping(
+        &mut self,
+        req: &PingRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut PingResponse,
+    ) -> anyhow::Result<()>;
+
+    fn supply_keypair(
+        &mut self,
+        req: &super::SupplyKeypairRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::SupplyKeypairResponse,
+    ) -> anyhow::Result<()>;
+
+    fn add_listen_socket(
+        &mut self,
+        req: &super::AddListenSocketRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::AddListenSocketResponse,
+    ) -> anyhow::Result<()>;
+
+    fn add_psk_broker(
+        &mut self,
+        req: &super::AddPskBrokerRequest,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: &mut super::AddPskBrokerResponse,
+    ) -> anyhow::Result<()>;
+
+    fn dispatch<ReqBuf, ResBuf>(
+        &mut self,
+        p: &mut RequestResponsePair<ReqBuf, ResBuf>,
+        req_fds: &mut VecDeque<OwnedFd>,
+    ) -> anyhow::Result<()>
+    where
+        ReqBuf: ByteSlice,
+        ResBuf: ByteSliceMut,
+    {
+        match p {
+            RequestResponsePair::Ping((req, res)) => self.ping(req, req_fds, res),
+            RequestResponsePair::SupplyKeypair((req, res)) => {
+                self.supply_keypair(req, req_fds, res)
+            }
+            RequestResponsePair::AddListenSocket((req, res)) => {
+                self.add_listen_socket(req, req_fds, res)
+            }
+            RequestResponsePair::AddPskBroker((req, res)) => self.add_psk_broker(req, req_fds, res),
+        }
+    }
+
+    fn handle_message<ReqBuf, ResBuf>(
+        &mut self,
+        req: ReqBuf,
+        req_fds: &mut VecDeque<OwnedFd>,
+        res: ResBuf,
+    ) -> anyhow::Result<usize>
+    where
+        ReqBuf: ByteSlice,
+        ResBuf: ByteSliceMut,
+    {
+        let req = req.parse_request_from_prefix()?;
+        // TODO: This is not pretty; This match should be moved into RequestRef
+        let mut pair = match req {
+            RequestRef::Ping(req) => {
+                let mut res = res.ping_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::Ping((req, res))
+            }
+            RequestRef::SupplyKeypair(req) => {
+                let mut res = res.supply_keypair_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::SupplyKeypair((req, res))
+            }
+            RequestRef::AddListenSocket(req) => {
+                let mut res = res.add_listen_socket_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::AddListenSocket((req, res))
+            }
+            RequestRef::AddPskBroker(req) => {
+                let mut res = res.add_psk_broker_response_from_prefix()?;
+                res.init();
+                RequestResponsePair::AddPskBroker((req, res))
+            }
+        };
+        self.dispatch(&mut pair, req_fds)?;
+
+        let res_len = pair.response().bytes().len();
+        Ok(res_len)
+    }
+}

rosenpass/src/api/cli.rs

@@ -0,0 +1,40 @@
+use std::path::PathBuf;
+
+use clap::Args;
+
+use crate::config::Rosenpass as RosenpassConfig;
+
+use super::config::ApiConfig;
+
+#[cfg(feature = "experiment_api")]
+#[derive(Args, Debug)]
+pub struct ApiCli {
+    /// Where in the file-system to create the unix socket the rosenpass API will be listening for
+    /// connections on.
+    #[arg(long)]
+    api_listen_path: Vec<PathBuf>,
+
+    /// When rosenpass is called from another process, the other process can open and bind the
+    /// unix socket for the Rosenpass API to use themselves, passing it to this process. In Rust this can be achieved
+    /// using the [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate.
+    #[arg(long)]
+    api_listen_fd: Vec<i32>,
+
+    /// When rosenpass is called from another process, the other process can connect the unix socket for the API
+    /// themselves, for instance using the `socketpair(2)` system call.
+    #[arg(long)]
+    api_stream_fd: Vec<i32>,
+}
+
+impl ApiCli {
+    pub fn apply_to_config(&self, cfg: &mut RosenpassConfig) -> anyhow::Result<()> {
+        self.apply_to_api_config(&mut cfg.api)
+    }
+
+    pub fn apply_to_api_config(&self, cfg: &mut ApiConfig) -> anyhow::Result<()> {
+        cfg.listen_path.extend_from_slice(&self.api_listen_path);
+        cfg.listen_fd.extend_from_slice(&self.api_listen_fd);
+        cfg.stream_fd.extend_from_slice(&self.api_stream_fd);
+        Ok(())
+    }
+}

rosenpass/src/api/config.rs

@@ -0,0 +1,49 @@
+use std::path::PathBuf;
+
+use mio::net::UnixListener;
+use rosenpass_util::mio::{UnixListenerExt, UnixStreamExt};
+use serde::{Deserialize, Serialize};
+
+use crate::app_server::AppServer;
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct ApiConfig {
+    /// Where in the file-system to create the unix socket the rosenpass API will be listening for
+    /// connections on
+    pub listen_path: Vec<PathBuf>,
+
+    /// When rosenpass is called from another process, the other process can open and bind the
+    /// unix socket for the Rosenpass API to use themselves, passing it to this process. In Rust this can be achieved
+    /// using the [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate.
+    pub listen_fd: Vec<i32>,
+
+    /// When rosenpass is called from another process, the other process can connect the unix socket for the API
+    /// themselves, for instance using the `socketpair(2)` system call.
+    pub stream_fd: Vec<i32>,
+}
+
+impl ApiConfig {
+    pub fn apply_to_app_server(&self, srv: &mut AppServer) -> anyhow::Result<()> {
+        for path in self.listen_path.iter() {
+            srv.add_api_listener(UnixListener::bind(path)?)?;
+        }
+
+        for fd in self.listen_fd.iter() {
+            srv.add_api_listener(UnixListenerExt::claim_fd(*fd)?)?;
+        }
+
+        for fd in self.stream_fd.iter() {
+            srv.add_api_connection(UnixStreamExt::claim_fd(*fd)?)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn count_api_sources(&self) -> usize {
+        self.listen_path.len() + self.listen_fd.len() + self.stream_fd.len()
+    }
+
+    pub fn has_api_sources(&self) -> bool {
+        self.count_api_sources() > 0
+    }
+}

rosenpass/src/api/mio/connection.rs

@@ -0,0 +1,321 @@
+use std::borrow::{Borrow, BorrowMut};
+use std::collections::VecDeque;
+use std::os::fd::OwnedFd;
+
+use mio::net::UnixStream;
+use rosenpass_secret_memory::Secret;
+use rosenpass_util::mio::ReadWithFileDescriptors;
+use rosenpass_util::{
+    io::{IoResultKindHintExt, TryIoResultKindHintExt},
+    length_prefix_encoding::{
+        decoder::{self as lpe_decoder, LengthPrefixDecoder},
+        encoder::{self as lpe_encoder, LengthPrefixEncoder},
+    },
+    mio::interest::RW as MIO_RW,
+};
+use zeroize::Zeroize;
+
+use crate::api::MAX_REQUEST_FDS;
+use crate::{api::Server, app_server::AppServer};
+
+use super::super::{ApiHandler, ApiHandlerContext};
+
+#[derive(Debug)]
+struct SecretBuffer<const N: usize>(pub Secret<N>);
+
+impl<const N: usize> SecretBuffer<N> {
+    fn new() -> Self {
+        Self(Secret::zero())
+    }
+}
+
+impl<const N: usize> Borrow<[u8]> for SecretBuffer<N> {
+    fn borrow(&self) -> &[u8] {
+        self.0.secret()
+    }
+}
+
+impl<const N: usize> BorrowMut<[u8]> for SecretBuffer<N> {
+    fn borrow_mut(&mut self) -> &mut [u8] {
+        self.0.secret_mut()
+    }
+}
+// TODO: Unfortunately, zerocopy is quite particular about alignment, hence the 4096
+type ReadBuffer = LengthPrefixDecoder<SecretBuffer<4096>>;
+type WriteBuffer = LengthPrefixEncoder<SecretBuffer<4096>>;
+type ReadFdBuffer = VecDeque<OwnedFd>;
+
+#[derive(Debug)]
+struct MioConnectionBuffers {
+    read_buffer: ReadBuffer,
+    write_buffer: WriteBuffer,
+    read_fd_buffer: ReadFdBuffer,
+}
+
+#[derive(Debug)]
+pub struct MioConnection {
+    io: UnixStream,
+    mio_token: mio::Token,
+    invalid_read: bool,
+    buffers: Option<MioConnectionBuffers>,
+    api_handler: ApiHandler,
+}
+
+impl MioConnection {
+    pub fn new(app_server: &mut AppServer, mut io: UnixStream) -> std::io::Result<Self> {
+        let mio_token = app_server.mio_token_dispenser.dispense();
+        app_server
+            .mio_poll
+            .registry()
+            .register(&mut io, mio_token, MIO_RW)?;
+
+        let invalid_read = false;
+        let read_buffer = LengthPrefixDecoder::new(SecretBuffer::new());
+        let write_buffer = LengthPrefixEncoder::from_buffer(SecretBuffer::new());
+        let read_fd_buffer = VecDeque::new();
+        let buffers = Some(MioConnectionBuffers {
+            read_buffer,
+            write_buffer,
+            read_fd_buffer,
+        });
+        let api_state = ApiHandler::new();
+        Ok(Self {
+            io,
+            mio_token,
+            invalid_read,
+            buffers,
+            api_handler: api_state,
+        })
+    }
+
+    pub fn shoud_close(&self) -> bool {
+        let exhausted = self
+            .buffers
+            .as_ref()
+            .map(|b| b.write_buffer.exhausted())
+            .unwrap_or(false);
+        self.invalid_read && exhausted
+    }
+
+    pub fn close(mut self, app_server: &mut AppServer) -> anyhow::Result<()> {
+        app_server.mio_poll.registry().deregister(&mut self.io)?;
+        Ok(())
+    }
+
+    pub fn mio_token(&self) -> mio::Token {
+        self.mio_token
+    }
+}
+
+pub trait MioConnectionContext {
+    fn mio_connection(&self) -> &MioConnection;
+    fn app_server(&self) -> &AppServer;
+    fn mio_connection_mut(&mut self) -> &mut MioConnection;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+
+    fn poll(&mut self) -> anyhow::Result<()> {
+        macro_rules! short {
+            ($e:expr) => {
+                match $e {
+                    None => return Ok(()),
+                    Some(()) => {}
+                }
+            };
+        }
+
+        // All of these functions return an error, None ("operation incomplete")
+        // or some ("operation complete, keep processing")
+        short!(self.flush_write_buffer()?); // Flush last message
+        short!(self.recv()?); // Receive new message
+        short!(self.handle_incoming_message()?); // Process new message with API
+        short!(self.flush_write_buffer()?); // Begin flushing response
+
+        Ok(())
+    }
+
+    fn handle_incoming_message(&mut self) -> anyhow::Result<Option<()>> {
+        self.with_buffers_stolen(|this, bufs| {
+            // Acquire request & response. Caller is responsible to make sure
+            // that read buffer holds a message and that write buffer is cleared.
+            // Hence the unwraps and assertions
+            assert!(bufs.write_buffer.exhausted());
+            let req = bufs.read_buffer.message().unwrap().unwrap();
+            let req_fds = &mut bufs.read_fd_buffer;
+            let res = bufs.write_buffer.buffer_bytes_mut();
+
+            // Call API handler
+            // Transitive trait implementations: MioConnectionContext -> ApiHandlerContext -> as ApiServer
+            let response_len = this.handle_message(req, req_fds, res)?;
+
+            bufs.write_buffer
+                .restart_write_with_new_message(response_len)?;
+            bufs.read_buffer.zeroize(); // clear for new message to read
+            bufs.read_fd_buffer.clear();
+
+            Ok(Some(()))
+        })
+    }
+
+    fn flush_write_buffer(&mut self) -> anyhow::Result<Option<()>> {
+        if self.write_buf_mut().exhausted() {
+            return Ok(Some(()));
+        }
+
+        use lpe_encoder::WriteToIoReturn as Ret;
+        use std::io::ErrorKind as K;
+
+        loop {
+            let conn = self.mio_connection_mut();
+            let bufs = conn.buffers.as_mut().unwrap();
+
+            let sock = &conn.io;
+            let write_buf = &mut bufs.write_buffer;
+
+            match write_buf.write_to_stdio(sock).io_err_kind_hint() {
+                // Done
+                Ok(Ret { done: true, .. }) => {
+                    write_buf.zeroize(); // clear for new message to write
+                    break Ok(Some(()));
+                }
+
+                // Would block
+                Ok(Ret {
+                    bytes_written: 0, ..
+                }) => break Ok(None),
+                Err((_e, K::WouldBlock)) => break Ok(None),
+
+                // Just continue
+                Ok(_) => continue, /* Ret { bytes_written > 0, done = false } acc. to previous cases*/
+                Err((_e, K::Interrupted)) => continue,
+
+                // Other errors
+                Err((e, _ek)) => Err(e)?,
+            }
+        }
+    }
+
+    fn recv(&mut self) -> anyhow::Result<Option<()>> {
+        if !self.write_buf_mut().exhausted() || self.mio_connection().invalid_read {
+            return Ok(None);
+        }
+
+        use lpe_decoder::{ReadFromIoError as E, ReadFromIoReturn as Ret};
+        use std::io::ErrorKind as K;
+
+        loop {
+            let conn = self.mio_connection_mut();
+            let bufs = conn.buffers.as_mut().unwrap();
+
+            let read_buf = &mut bufs.read_buffer;
+            let read_fd_buf = &mut bufs.read_fd_buffer;
+
+            let sock = &conn.io;
+            let fd_passing_sock = ReadWithFileDescriptors::<MAX_REQUEST_FDS, UnixStream, _, _>::new(
+                sock,
+                read_fd_buf,
+            );
+
+            match read_buf
+                .read_from_stdio(fd_passing_sock)
+                .try_io_err_kind_hint()
+            {
+                // We actually received a proper message
+                // (Impl below match to appease borrow checker)
+                Ok(Ret {
+                    message: Some(_msg),
+                    ..
+                }) => break Ok(Some(())),
+
+                // Message does not fit in buffer
+                Err((e @ E::MessageTooLargeError { .. }, _)) => {
+                    log::warn!("Received message on API that was too big to fit in our buffers; \
+                            looks like the client is broken. Stopping to process messages of the client.\n\
+                            Error: {e:?}");
+                    conn.invalid_read = true; // Closed mio_manager
+                    break Ok(None);
+                }
+
+                // Would block
+                Ok(Ret { bytes_read: 0, .. }) => break Ok(None),
+                Err((_, Some(K::WouldBlock))) => break Ok(None),
+
+                // Just keep going
+                Ok(Ret { bytes_read: _, .. }) => continue,
+                Err((_, Some(K::Interrupted))) => continue,
+
+                // Other IO Error (just pass on to the caller)
+                Err((E::IoError(e), _)) => {
+                    log::warn!(
+                        "IO error while trying to read message from API socket. \
+                            The connection is broken. Stopping to process messages of the client.\n\
+                            Error: {e:?}"
+                    );
+                    conn.invalid_read = true; // closed later by mio_manager
+                    break Err(e.into());
+                }
+            };
+        }
+    }
+
+    fn mio_token(&self) -> mio::Token {
+        self.mio_connection().mio_token()
+    }
+
+    fn should_close(&self) -> bool {
+        self.mio_connection().shoud_close()
+    }
+}
+
+trait MioConnectionContextPrivate: MioConnectionContext {
+    fn steal_buffers(&mut self) -> MioConnectionBuffers {
+        self.mio_connection_mut().buffers.take().unwrap()
+    }
+
+    fn return_buffers(&mut self, buffers: MioConnectionBuffers) {
+        let opt = &mut self.mio_connection_mut().buffers;
+        assert!(opt.is_none());
+        let _ = opt.insert(buffers);
+    }
+
+    fn with_buffers_stolen<R, F: FnOnce(&mut Self, &mut MioConnectionBuffers) -> R>(
+        &mut self,
+        f: F,
+    ) -> R {
+        let mut bufs = self.steal_buffers();
+        let res = f(self, &mut bufs);
+        self.return_buffers(bufs);
+        res
+    }
+
+    fn write_buf_mut(&mut self) -> &mut WriteBuffer {
+        self.mio_connection_mut()
+            .buffers
+            .as_mut()
+            .unwrap()
+            .write_buffer
+            .borrow_mut()
+    }
+}
+
+impl<T> MioConnectionContextPrivate for T where T: ?Sized + MioConnectionContext {}
+
+impl<T> ApiHandlerContext for T
+where
+    T: ?Sized + MioConnectionContext,
+{
+    fn api_handler(&self) -> &ApiHandler {
+        &self.mio_connection().api_handler
+    }
+
+    fn app_server(&self) -> &AppServer {
+        MioConnectionContext::app_server(self)
+    }
+
+    fn api_handler_mut(&mut self) -> &mut ApiHandler {
+        &mut self.mio_connection_mut().api_handler
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        MioConnectionContext::app_server_mut(self)
+    }
+}

rosenpass/src/api/mio/manager.rs

@@ -0,0 +1,173 @@
+use std::{borrow::BorrowMut, io};
+
+use mio::net::{UnixListener, UnixStream};
+
+use rosenpass_util::{
+    functional::ApplyExt, io::nonblocking_handle_io_errors, mio::interest::RW as MIO_RW,
+};
+
+use crate::app_server::{AppServer, AppServerIoSource};
+
+use super::{MioConnection, MioConnectionContext};
+
+#[derive(Default, Debug)]
+pub struct MioManager {
+    listeners: Vec<UnixListener>,
+    connections: Vec<Option<MioConnection>>,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum MioManagerIoSource {
+    Listener(usize),
+    Connection(usize),
+}
+
+impl MioManager {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+struct MioConnectionFocus<'a, T: ?Sized + MioManagerContext> {
+    ctx: &'a mut T,
+    conn_idx: usize,
+}
+
+impl<'a, T: ?Sized + MioManagerContext> MioConnectionFocus<'a, T> {
+    fn new(ctx: &'a mut T, conn_idx: usize) -> Self {
+        Self { ctx, conn_idx }
+    }
+}
+
+pub trait MioManagerContext {
+    fn mio_manager(&self) -> &MioManager;
+    fn mio_manager_mut(&mut self) -> &mut MioManager;
+    fn app_server(&self) -> &AppServer;
+    fn app_server_mut(&mut self) -> &mut AppServer;
+
+    fn add_listener(&mut self, mut listener: UnixListener) -> io::Result<()> {
+        let srv = self.app_server_mut();
+        let mio_token = srv.mio_token_dispenser.dispense();
+        srv.mio_poll
+            .registry()
+            .register(&mut listener, mio_token, MIO_RW)?;
+        let io_source = self
+            .mio_manager()
+            .listeners
+            .len()
+            .apply(MioManagerIoSource::Listener)
+            .apply(AppServerIoSource::MioManager);
+        self.mio_manager_mut().listeners.push(listener);
+        self.app_server_mut()
+            .register_io_source(mio_token, io_source);
+
+        Ok(())
+    }
+
+    fn add_connection(&mut self, connection: UnixStream) -> io::Result<()> {
+        let connection = MioConnection::new(self.app_server_mut(), connection)?;
+        let mio_token = connection.mio_token();
+        let conns: &mut Vec<Option<MioConnection>> =
+            self.mio_manager_mut().connections.borrow_mut();
+        let idx = conns
+            .iter_mut()
+            .enumerate()
+            .find(|(_, slot)| slot.is_some())
+            .map(|(idx, _)| idx)
+            .unwrap_or(conns.len());
+        conns.insert(idx, Some(connection));
+        let io_source = idx
+            .apply(MioManagerIoSource::Listener)
+            .apply(AppServerIoSource::MioManager);
+        self.app_server_mut()
+            .register_io_source(mio_token, io_source);
+        Ok(())
+    }
+
+    fn poll_particular(&mut self, io_source: MioManagerIoSource) -> anyhow::Result<()> {
+        use MioManagerIoSource as S;
+        match io_source {
+            S::Listener(idx) => self.accept_from(idx)?,
+            S::Connection(idx) => self.poll_particular_connection(idx)?,
+        };
+        Ok(())
+    }
+
+    fn poll(&mut self) -> anyhow::Result<()> {
+        self.accept_connections()?;
+        self.poll_connections()?;
+        Ok(())
+    }
+
+    fn accept_connections(&mut self) -> io::Result<()> {
+        for idx in 0..self.mio_manager_mut().listeners.len() {
+            self.accept_from(idx)?;
+        }
+        Ok(())
+    }
+
+    fn accept_from(&mut self, idx: usize) -> io::Result<()> {
+        // Accept connection until the socket would block or returns another error
+        // TODO: This currently only adds connections--we eventually need the ability to remove
+        // them as well, see the note in connection.rs
+        loop {
+            match nonblocking_handle_io_errors(|| self.mio_manager().listeners[idx].accept())? {
+                None => break,
+                Some((conn, _addr)) => {
+                    self.add_connection(conn)?;
+                }
+            };
+        }
+
+        Ok(())
+    }
+
+    fn poll_connections(&mut self) -> anyhow::Result<()> {
+        for idx in 0..self.mio_manager().connections.len() {
+            self.poll_particular_connection(idx)?;
+        }
+        Ok(())
+    }
+
+    fn poll_particular_connection(&mut self, idx: usize) -> anyhow::Result<()> {
+        if self.mio_manager().connections[idx].is_none() {
+            return Ok(());
+        }
+
+        let mut conn = MioConnectionFocus::new(self, idx);
+        conn.poll()?;
+
+        if conn.should_close() {
+            let conn = self.mio_manager_mut().connections[idx].take().unwrap();
+            let mio_token = conn.mio_token();
+            if let Err(e) = conn.close(self.app_server_mut()) {
+                log::warn!("Error while closing API connection {e:?}");
+            };
+            self.app_server_mut().unregister_io_source(mio_token);
+        }
+
+        Ok(())
+    }
+}
+
+impl<T: ?Sized + MioManagerContext> MioConnectionContext for MioConnectionFocus<'_, T> {
+    fn mio_connection(&self) -> &MioConnection {
+        self.ctx.mio_manager().connections[self.conn_idx]
+            .as_ref()
+            .unwrap()
+    }
+
+    fn app_server(&self) -> &AppServer {
+        self.ctx.app_server()
+    }
+
+    fn mio_connection_mut(&mut self) -> &mut MioConnection {
+        self.ctx.mio_manager_mut().connections[self.conn_idx]
+            .as_mut()
+            .unwrap()
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        self.ctx.app_server_mut()
+    }
+}

rosenpass/src/api/mio/mod.rs

@@ -0,0 +1,5 @@
+mod connection;
+mod manager;
+
+pub use connection::*;
+pub use manager::*;

rosenpass/src/api/mod.rs

@@ -0,0 +1,9 @@
+mod api_handler;
+mod boilerplate;
+
+pub use api_handler::*;
+pub use boilerplate::*;
+
+pub mod cli;
+pub mod config;
+pub mod mio;

rosenpass/src/app_server.rs

@@ -1,5 +1,6 @@
 use anyhow::bail;
 
+use anyhow::Context;
 use anyhow::Result;
 use derive_builder::Builder;
 use log::{error, info, warn};
@@ -7,7 +8,14 @@ use mio::Interest;
 use mio::Token;
 use rosenpass_secret_memory::Public;
 use rosenpass_secret_memory::Secret;
+use rosenpass_util::build::ConstructionSite;
 use rosenpass_util::file::StoreValueB64;
+use rosenpass_util::functional::run;
+use rosenpass_util::functional::ApplyExt;
+use rosenpass_util::io::IoResultKindHintExt;
+use rosenpass_util::io::SubstituteForIoErrorKindExt;
+use rosenpass_util::option::SomeExt;
+use rosenpass_util::result::OkExt;
 use rosenpass_wireguard_broker::WireguardBrokerMio;
 use rosenpass_wireguard_broker::{WireguardBrokerCfg, WG_KEY_LEN};
 use zerocopy::AsBytes;
@@ -15,8 +23,12 @@ use zerocopy::AsBytes;
 use std::cell::Cell;
 
 use std::collections::HashMap;
+use std::collections::VecDeque;
 use std::fmt::Debug;
+use std::io;
+use std::io::stdout;
 use std::io::ErrorKind;
+use std::io::Write;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
@@ -28,6 +40,7 @@ use std::slice;
 use std::time::Duration;
 use std::time::Instant;
 
+use crate::protocol::BuildCryptoServer;
 use crate::protocol::HostIdentification;
 use crate::{
     config::Verbosity,
@@ -63,7 +76,7 @@ pub struct MioTokenDispenser {
 }
 
 impl MioTokenDispenser {
-    fn dispense(&mut self) -> Token {
+    pub fn dispense(&mut self) -> Token {
         let r = self.counter;
         self.counter += 1;
         Token(r)
@@ -72,7 +85,7 @@ impl MioTokenDispenser {
 
 #[derive(Debug, Default)]
 pub struct BrokerStore {
-    store: HashMap<
+    pub store: HashMap<
         Public<BROKER_ID_BYTES>,
         Box<dyn WireguardBrokerMio<Error = anyhow::Error, MioError = anyhow::Error>>,
     >,
@@ -138,15 +151,29 @@ pub struct AppServerTest {
     pub termination_handler: Option<std::sync::mpsc::Receiver<()>>,
 }
 
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum AppServerIoSource {
+    Socket(usize),
+    #[cfg(feature = "experiment_api")]
+    PskBroker(Public<BROKER_ID_BYTES>),
+    #[cfg(feature = "experiment_api")]
+    MioManager(crate::api::mio::MioManagerIoSource),
+}
+
+const EVENT_CAPACITY: usize = 20;
+
 /// Holds the state of the application, namely the external IO
 ///
 /// Responsible for file IO, network IO
 // TODO add user control via unix domain socket and stdin/stdout
 #[derive(Debug)]
 pub struct AppServer {
-    pub crypt: CryptoServer,
+    pub crypto_site: ConstructionSite<BuildCryptoServer, CryptoServer>,
     pub sockets: Vec<mio::net::UdpSocket>,
     pub events: mio::Events,
+    pub short_poll_queue: VecDeque<mio::event::Event>,
+    pub performed_long_poll: bool,
+    pub io_source_index: HashMap<mio::Token, AppServerIoSource>,
     pub mio_poll: mio::Poll,
     pub mio_token_dispenser: MioTokenDispenser,
     pub brokers: BrokerStore,
@@ -159,6 +186,8 @@ pub struct AppServer {
     pub unpolled_count: usize,
     pub last_update_time: Instant,
     pub test_helpers: Option<AppServerTest>,
+    #[cfg(feature = "experiment_api")]
+    pub api_manager: crate::api::mio::MioManager,
 }
 
 /// A socket pointer is an index assigned to a socket;
@@ -507,15 +536,14 @@ impl HostPathDiscoveryEndpoint {
 
 impl AppServer {
     pub fn new(
-        sk: SSk,
-        pk: SPk,
+        keypair: Option<(SSk, SPk)>,
         addrs: Vec<SocketAddr>,
         verbosity: Verbosity,
         test_helpers: Option<AppServerTest>,
     ) -> anyhow::Result<Self> {
         // setup mio
         let mio_poll = mio::Poll::new()?;
-        let events = mio::Events::with_capacity(20);
+        let events = mio::Events::with_capacity(EVENT_CAPACITY);
         let mut mio_token_dispenser = MioTokenDispenser::default();
 
         // bind each SocketAddr to a socket
@@ -590,22 +618,30 @@ impl AppServer {
         }
 
         // register all sockets to mio
-        for socket in sockets.iter_mut() {
-            mio_poll.registry().register(
-                socket,
-                mio_token_dispenser.dispense(),
-                Interest::READABLE,
-            )?;
+        let mut io_source_index = HashMap::new();
+        for (idx, socket) in sockets.iter_mut().enumerate() {
+            let mio_token = mio_token_dispenser.dispense();
+            mio_poll
+                .registry()
+                .register(socket, mio_token, Interest::READABLE)?;
+            let prev = io_source_index.insert(mio_token, AppServerIoSource::Socket(idx));
+            assert!(prev.is_none());
         }
 
-        // TODO use mio::net::UnixStream together with std::os::unix::net::UnixStream for Linux
+        let crypto_site = match keypair {
+            Some((sk, pk)) => ConstructionSite::from_product(CryptoServer::new(sk, pk)),
+            None => ConstructionSite::new(BuildCryptoServer::empty()),
+        };
 
         Ok(Self {
-            crypt: CryptoServer::new(sk, pk),
+            crypto_site,
             peers: Vec::new(),
             verbosity,
             sockets,
             events,
+            short_poll_queue: Default::default(),
+            performed_long_poll: false,
+            io_source_index,
             mio_poll,
             mio_token_dispenser,
             brokers: BrokerStore::default(),
@@ -616,48 +652,78 @@ impl AppServer {
             unpolled_count: 0,
             last_update_time: Instant::now(),
             test_helpers,
+            #[cfg(feature = "experiment_api")]
+            api_manager: crate::api::mio::MioManager::default(),
         })
     }
 
+    pub fn crypto_server(&self) -> anyhow::Result<&CryptoServer> {
+        self.crypto_site
+            .product_ref()
+            .context("Cryptography handler not initialized")
+    }
+
+    pub fn crypto_server_mut(&mut self) -> anyhow::Result<&mut CryptoServer> {
+        self.crypto_site
+            .product_mut()
+            .context("Cryptography handler not initialized")
+    }
+
     pub fn verbose(&self) -> bool {
         matches!(self.verbosity, Verbosity::Verbose)
     }
 
+    pub fn register_listen_socket(&mut self, mut sock: mio::net::UdpSocket) -> anyhow::Result<()> {
+        let mio_token = self.mio_token_dispenser.dispense();
+        self.mio_poll
+            .registry()
+            .register(&mut sock, mio_token, mio::Interest::READABLE)?;
+        let io_source = self.sockets.len().apply(AppServerIoSource::Socket);
+        self.sockets.push(sock);
+        self.register_io_source(mio_token, io_source);
+        Ok(())
+    }
+
+    pub fn register_io_source(&mut self, token: mio::Token, io_source: AppServerIoSource) {
+        let prev = self.io_source_index.insert(token, io_source);
+        assert!(prev.is_none());
+    }
+
+    pub fn unregister_io_source(&mut self, token: mio::Token) {
+        let value = self.io_source_index.remove(&token);
+        assert!(value.is_some(), "Removed IO source that does not exist");
+    }
+
     pub fn register_broker(
         &mut self,
         broker: Box<dyn WireguardBrokerMio<Error = anyhow::Error, MioError = anyhow::Error>>,
     ) -> Result<BrokerStorePtr> {
         let ptr = Public::from_slice((self.brokers.store.len() as u64).as_bytes());
-
         if self.brokers.store.insert(ptr, broker).is_some() {
             bail!("Broker already registered");
         }
+
+        let mio_token = self.mio_token_dispenser.dispense();
+        let io_source = ptr.apply(AppServerIoSource::PskBroker);
         //Register broker
         self.brokers
             .store
             .get_mut(&ptr)
             .ok_or(anyhow::format_err!("Broker wasn't added to registry"))?
-            .register(
-                self.mio_poll.registry(),
-                self.mio_token_dispenser.dispense(),
-            )?;
+            .register(self.mio_poll.registry(), mio_token)?;
+        self.register_io_source(mio_token, io_source);
 
         Ok(BrokerStorePtr(ptr))
     }
 
     pub fn unregister_broker(&mut self, ptr: BrokerStorePtr) -> Result<()> {
-        //Unregister broker
-        self.brokers
-            .store
-            .get_mut(&ptr.0)
-            .ok_or_else(|| anyhow::anyhow!("Broker not found"))?
-            .unregister(self.mio_poll.registry())?;
-
-        //Remove broker from store
-        self.brokers
+        let mut broker = self
+            .brokers
             .store
             .remove(&ptr.0)
-            .ok_or_else(|| anyhow::anyhow!("Broker not found"))?;
+            .context("Broker not found")?;
+        self.unregister_io_source(broker.mio_token().unwrap());
+        broker.unregister(self.mio_poll.registry())?;
         Ok(())
     }
 
@@ -669,8 +735,13 @@ impl AppServer {
         broker_peer: Option<BrokerPeer>,
         hostname: Option<String>,
     ) -> anyhow::Result<AppPeerPtr> {
-        let PeerPtr(pn) = self.crypt.add_peer(psk, pk)?;
+        let PeerPtr(pn) = match &mut self.crypto_site {
+            ConstructionSite::Void => bail!("Crypto server construction site is void"),
+            ConstructionSite::Builder(builder) => builder.add_peer(psk, pk),
+            ConstructionSite::Product(srv) => srv.add_peer(psk, pk)?,
+        };
         assert!(pn == self.peers.len());
+
         let initial_endpoint = hostname
             .map(Endpoint::discovery_from_hostname)
             .transpose()?;
@@ -712,7 +783,7 @@ impl AppServer {
             );
             if tries_left > 0 {
                 error!("re-initializing networking in {sleep}! {tries_left} tries left.");
-                std::thread::sleep(self.crypt.timebase.dur(sleep));
+                std::thread::sleep(Duration::from_secs_f64(sleep));
                 continue;
             }
 
@@ -755,16 +826,31 @@ impl AppServer {
                 }
             }
 
-            match self.poll(&mut *rx)? {
+            enum CryptoSrv {
+                Avail,
+                Missing,
+            }
+
+            let poll_result = self.poll(&mut *rx)?;
+            let have_crypto = match self.crypto_site.is_available() {
+                true => CryptoSrv::Avail,
+                false => CryptoSrv::Missing,
+            };
+
             #[allow(clippy::redundant_closure_call)]
-                SendInitiation(peer) => tx_maybe_with!(peer, || self
-                    .crypt
+            match (have_crypto, poll_result) {
+                (CryptoSrv::Missing, SendInitiation(_)) => {}
+                (CryptoSrv::Avail, SendInitiation(peer)) => tx_maybe_with!(peer, || self
+                    .crypto_server_mut()?
                     .initiate_handshake(peer.lower(), &mut *tx))?,
-                #[allow(clippy::redundant_closure_call)]
-                SendRetransmission(peer) => tx_maybe_with!(peer, || self
-                    .crypt
+
+                (CryptoSrv::Missing, SendRetransmission(_)) => {}
+                (CryptoSrv::Avail, SendRetransmission(peer)) => tx_maybe_with!(peer, || self
+                    .crypto_server_mut()?
                     .retransmit_handshake(peer.lower(), &mut *tx))?,
-                DeleteKey(peer) => {
+
+                (CryptoSrv::Missing, DeleteKey(_)) => {}
+                (CryptoSrv::Avail, DeleteKey(peer)) => {
                     self.output_key(peer, Stale, &SymKey::random())?;
 
                     // There was a loss of connection apparently; restart host discovery
@@ -778,12 +864,15 @@ impl AppServer {
                     );
                 }
 
-                ReceivedMessage(len, endpoint) => {
+                (CryptoSrv::Missing, ReceivedMessage(_, _)) => {}
+                (CryptoSrv::Avail, ReceivedMessage(len, endpoint)) => {
                     let msg_result = match self.under_load {
                         DoSOperation::UnderLoad => {
                             self.handle_msg_under_load(&endpoint, &rx[..len], &mut *tx)
                         }
-                        DoSOperation::Normal => self.crypt.handle_msg(&rx[..len], &mut *tx),
+                        DoSOperation::Normal => {
+                            self.crypto_server_mut()?.handle_msg(&rx[..len], &mut *tx)
+                        }
                     };
                     match msg_result {
                         Err(ref e) => {
@@ -811,7 +900,8 @@ impl AppServer {
                                 ap.get_app_mut(self).current_endpoint = Some(endpoint);
 
                                 // TODO: Maybe we should rather call the key "rosenpass output"?
-                                self.output_key(ap, Exchanged, &self.crypt.osk(p)?)?;
+                                let osk = &self.crypto_server_mut()?.osk(p)?;
+                                self.output_key(ap, Exchanged, osk)?;
                             }
                         }
                     }
@@ -827,9 +917,9 @@ impl AppServer {
         tx: &mut [u8],
     ) -> Result<crate::protocol::HandleMsgResult> {
         match endpoint {
-            Endpoint::SocketBoundAddress(socket) => {
-                self.crypt.handle_msg_under_load(rx, &mut *tx, socket)
-            }
+            Endpoint::SocketBoundAddress(socket) => self
+                .crypto_server_mut()?
+                .handle_msg_under_load(rx, &mut *tx, socket),
             Endpoint::Discovery(_) => {
                 anyhow::bail!("Host-path discovery is not supported under load")
             }
@@ -842,7 +932,7 @@ impl AppServer {
         why: KeyOutputReason,
         key: &SymKey,
     ) -> anyhow::Result<()> {
-        let peerid = peer.lower().get(&self.crypt).pidt()?;
+        let peerid = peer.lower().get(self.crypto_server()?).pidt()?;
 
         if self.verbose() {
             let msg = match why {
@@ -870,10 +960,14 @@ impl AppServer {
 
             // this is intentionally writing to stdout instead of stderr, because
             // it is meant to allow external detection of a successful key-exchange
-            println!(
+            let stdout = stdout();
+            let mut stdout = stdout.lock();
+            writeln!(
+                stdout,
                 "output-key peer {} key-file {of:?} {why}",
                 peerid.fmt_b64::<MAX_B64_PEER_ID_SIZE>()
-            );
+            )?;
+            stdout.flush()?;
         }
 
         peer.set_psk(self, key)?;
@@ -884,17 +978,32 @@ impl AppServer {
     pub fn poll(&mut self, rx_buf: &mut [u8]) -> anyhow::Result<AppPollResult> {
         use crate::protocol::PollResult as C;
         use AppPollResult as A;
-        loop {
-            return Ok(match self.crypt.poll()? {
-                C::DeleteKey(PeerPtr(no)) => A::DeleteKey(AppPeerPtr(no)),
-                C::SendInitiation(PeerPtr(no)) => A::SendInitiation(AppPeerPtr(no)),
-                C::SendRetransmission(PeerPtr(no)) => A::SendRetransmission(AppPeerPtr(no)),
-                C::Sleep(timeout) => match self.try_recv(rx_buf, timeout)? {
-                    Some((len, addr)) => A::ReceivedMessage(len, addr),
-                    None => continue,
-                },
-            });
+        let res = loop {
+            // Call CryptoServer's poll (if available)
+            let crypto_poll = self
+                .crypto_site
+                .product_mut()
+                .map(|crypto| crypto.poll())
+                .transpose()?;
+
+            // Map crypto server's poll result to our poll result
+            let io_poll_timeout = match crypto_poll {
+                Some(C::DeleteKey(PeerPtr(no))) => break A::DeleteKey(AppPeerPtr(no)),
+                Some(C::SendInitiation(PeerPtr(no))) => break A::SendInitiation(AppPeerPtr(no)),
+                Some(C::SendRetransmission(PeerPtr(no))) => {
+                    break A::SendRetransmission(AppPeerPtr(no))
                 }
+                Some(C::Sleep(timeout)) => timeout, // No event from crypto-server, do IO
+                None => crate::protocol::UNENDING,  // Crypto server is uninitialized, do IO
+            };
+
+            // Perform IO (look for a message)
+            if let Some((len, addr)) = self.try_recv(rx_buf, io_poll_timeout)? {
+                break A::ReceivedMessage(len, addr);
+            }
+        };
+
+        Ok(res)
     }
 
     /// Tries to receive a new message
@@ -932,23 +1041,34 @@ impl AppServer {
         // readiness event seems to be good enough™ for now.
 
         // only poll if we drained all sockets before
-        if self.all_sockets_drained {
-            //Non blocked polling
-            self.mio_poll
-                .poll(&mut self.events, Some(Duration::from_secs(0)))?;
-
-            if self.events.iter().peekable().peek().is_none() {
-                // if there are no events, then add to blocking poll count
-                self.blocking_polls_count += 1;
-                //Execute blocking poll
-                self.mio_poll.poll(&mut self.events, Some(timeout))?;
-            } else {
-                self.non_blocking_polls_count += 1;
-            }
-        } else {
+        run(|| -> anyhow::Result<()> {
+            if !self.all_sockets_drained || !self.short_poll_queue.is_empty() {
                 self.unpolled_count += 1;
+                return Ok(());
             }
 
+            self.perform_mio_poll_and_register_events(Duration::from_secs(0))?; // Non-blocking poll
+            if !self.short_poll_queue.is_empty() {
+                // Got some events in non-blocking mode
+                self.non_blocking_polls_count += 1;
+                return Ok(());
+            }
+
+            if !self.performed_long_poll {
+                // pass – go perform a full long poll before we enter blocking poll mode
+                // to make sure our experimental short poll feature did not miss any events
+                // due to being buggy.
+                return Ok(());
+            }
+
+            // Perform and register blocking poll
+            self.blocking_polls_count += 1;
+            self.perform_mio_poll_and_register_events(timeout)?;
+            self.performed_long_poll = false;
+
+            Ok(())
+        })?;
+
         if let Some(AppServerTest {
             enable_dos_permanently: true,
             ..
@@ -982,26 +1102,58 @@ impl AppServer {
             }
         }
 
+        // Focused polling – i.e. actually using mio::Token – is experimental for now.
+        // The reason for this is that we need to figure out how to integrate load detection
+        // and focused polling for one. Mio event-based polling also does not play nice with
+        // the current function signature and its reentrant design which is focused around receiving UDP socket packages
+        // for processing by the crypto protocol server.
+        // Besides that, there are also some parts of the code which intentionally block
+        // despite available data. This is the correct behavior; e.g. api::mio::Connection blocks
+        // further reads from its unix socket until the write buffer is flushed. In other words
+        // the connection handler makes sure that there is a buffer to put the response in while
+        // before reading further request.
+        // The potential problem with this behavior is that we end up ignoring instructions from
+        // epoll() to read from the particular sockets, so epoll will return information about that
+        // particular – blocked – file descriptor every call. We have only so many event slots and
+        // in theory, the event array could fill up entirely with intentionally blocked sockets.
+        // We need to figure out how to deal with this situation.
+        // Mio uses uses epoll in level-triggered mode, so we could handle taint-tracking for ignored
+        // sockets ourselves. The facilities are available in epoll and Mio, but we need to figure out how mio uses those
+        // facilities and how we can integrate them here.
+        // This will involve rewriting a lot of IO code and we should probably have integration
+        // tests  before we approach that.
+        //
+        // This hybrid approach is not without merit though; the short poll implementation covers
+        // all our IO sources, so under contention, rosenpass should generally not hit the long
+        // poll mode below. We keep short polling and calling epoll() in non-blocking mode (timeout
+        // of zero) until we run out of IO events processed. Then, just before we would perform a
+        // blocking poll, we go through all available IO sources to see if we missed anything.
+        {
+            while let Some(ev) = self.short_poll_queue.pop_front() {
+                if let Some(v) = self.try_recv_from_mio_token(buf, ev.token())? {
+                    return Ok(Some(v));
+                }
+            }
+        }
+
         // drain all sockets
         let mut would_block_count = 0;
-        for (sock_no, socket) in self.sockets.iter_mut().enumerate() {
-            match socket.recv_from(buf) {
-                Ok((n, addr)) => {
+        for sock_no in 0..self.sockets.len() {
+            match self
+                .try_recv_from_listen_socket(buf, sock_no)
+                .io_err_kind_hint()
+            {
+                Ok(None) => continue,
+                Ok(Some(v)) => {
                     // at least one socket was not drained...
                     self.all_sockets_drained = false;
-                    return Ok(Some((
-                        n,
-                        Endpoint::SocketBoundAddress(SocketBoundEndpoint::new(
-                            SocketPtr(sock_no),
-                            addr,
-                        )),
-                    )));
+                    return Ok(Some(v));
                 }
-                Err(e) if e.kind() == ErrorKind::WouldBlock => {
+                Err((_, ErrorKind::WouldBlock)) => {
                     would_block_count += 1;
                 }
                 // TODO if one socket continuously returns an error, then we never poll, thus we never wait for a timeout, thus we have a spin-lock
-                Err(e) => return Err(e.into()),
+                Err((e, _)) => return Err(e)?,
             }
         }
 
@@ -1013,6 +1165,126 @@ impl AppServer {
             broker.process_poll()?;
         }
 
+        // API poll
+
+        #[cfg(feature = "experiment_api")]
+        {
+            use crate::api::mio::MioManagerContext;
+            MioManagerFocus(self).poll()?;
+        }
+
+        self.performed_long_poll = true;
+
         Ok(None)
     }
+
+    fn perform_mio_poll_and_register_events(&mut self, timeout: Duration) -> io::Result<()> {
+        self.mio_poll.poll(&mut self.events, Some(timeout))?;
+        // Fill the short poll buffer with the acquired events
+        self.events
+            .iter()
+            .cloned()
+            .for_each(|v| self.short_poll_queue.push_back(v));
+        Ok(())
+    }
+
+    fn try_recv_from_mio_token(
+        &mut self,
+        buf: &mut [u8],
+        token: mio::Token,
+    ) -> anyhow::Result<Option<(usize, Endpoint)>> {
+        let io_source = match self.io_source_index.get(&token) {
+            Some(io_source) => *io_source,
+            None => {
+                log::warn!("No IO source assiociated with mio token ({token:?}). Polling using mio tokens directly is an experimental feature and IO handler should recover when all available io sources are polled. This is a developer error. Please report it.");
+                return Ok(None);
+            }
+        };
+
+        self.try_recv_from_io_source(buf, io_source)
+    }
+
+    fn try_recv_from_io_source(
+        &mut self,
+        buf: &mut [u8],
+        io_source: AppServerIoSource,
+    ) -> anyhow::Result<Option<(usize, Endpoint)>> {
+        use crate::api::mio::MioManagerContext;
+
+        match io_source {
+            AppServerIoSource::Socket(idx) => self
+                .try_recv_from_listen_socket(buf, idx)
+                .substitute_for_ioerr_wouldblock(None)?
+                .ok(),
+
+            #[cfg(feature = "experiment_api")]
+            AppServerIoSource::PskBroker(key) => self
+                .brokers
+                .store
+                .get_mut(&key)
+                .with_context(|| format!("No PSK broker under key {key:?}"))?
+                .process_poll()
+                .map(|_| None),
+
+            #[cfg(feature = "experiment_api")]
+            AppServerIoSource::MioManager(mmio_src) => MioManagerFocus(self)
+                .poll_particular(mmio_src)
+                .map(|_| None),
+        }
+    }
+
+    fn try_recv_from_listen_socket(
+        &mut self,
+        buf: &mut [u8],
+        idx: usize,
+    ) -> io::Result<Option<(usize, Endpoint)>> {
+        use std::io::ErrorKind as K;
+        let (n, addr) = loop {
+            match self.sockets[idx].recv_from(buf).io_err_kind_hint() {
+                Ok(v) => break v,
+                Err((_, K::Interrupted)) => continue,
+                Err((e, _)) => return Err(e)?,
+            }
+        };
+        SocketPtr(idx)
+            .apply(|sp| SocketBoundEndpoint::new(sp, addr))
+            .apply(Endpoint::SocketBoundAddress)
+            .apply(|ep| (n, ep))
+            .some()
+            .ok()
+    }
+
+    #[cfg(feature = "experiment_api")]
+    pub fn add_api_connection(&mut self, connection: mio::net::UnixStream) -> std::io::Result<()> {
+        use crate::api::mio::MioManagerContext;
+        MioManagerFocus(self).add_connection(connection)
+    }
+
+    #[cfg(feature = "experiment_api")]
+    pub fn add_api_listener(&mut self, listener: mio::net::UnixListener) -> std::io::Result<()> {
+        use crate::api::mio::MioManagerContext;
+        MioManagerFocus(self).add_listener(listener)
+    }
+}
+
+#[cfg(feature = "experiment_api")]
+struct MioManagerFocus<'a>(&'a mut AppServer);
+
+#[cfg(feature = "experiment_api")]
+impl crate::api::mio::MioManagerContext for MioManagerFocus<'_> {
+    fn mio_manager(&self) -> &crate::api::mio::MioManager {
+        &self.0.api_manager
+    }
+
+    fn mio_manager_mut(&mut self) -> &mut crate::api::mio::MioManager {
+        &mut self.0.api_manager
+    }
+
+    fn app_server(&self) -> &AppServer {
+        self.0
+    }
+
+    fn app_server_mut(&mut self) -> &mut AppServer {
+        self.0
+    }
 }

rosenpass/src/bin/gen-ipc-msg-types.rs

@@ -0,0 +1,92 @@
+use anyhow::{Context, Result};
+use heck::ToShoutySnakeCase;
+
+use rosenpass_ciphers::{hash_domain::HashDomain, KEY_LEN};
+
+fn calculate_hash_value(hd: HashDomain, values: &[&str]) -> Result<[u8; KEY_LEN]> {
+    match values.split_first() {
+        Some((head, tail)) => calculate_hash_value(hd.mix(head.as_bytes())?, tail),
+        None => Ok(hd.into_value()),
+    }
+}
+
+fn print_literal(path: &[&str]) -> Result<()> {
+    let val = calculate_hash_value(HashDomain::zero(), path)?;
+    let (last, prefix) = path.split_last().context("developer error!")?;
+    let var_name = last.to_shouty_snake_case();
+
+    print!("// hash domain hash of: ");
+    for n in prefix.iter() {
+        print!("{n} -> ");
+    }
+    println!("{last}");
+
+    let c = hex::encode(val)
+        .chars()
+        .collect::<Vec<char>>()
+        .chunks_exact(4)
+        .map(|chunk| chunk.iter().collect::<String>())
+        .collect::<Vec<_>>();
+    println!("const {var_name} : RawMsgType = RawMsgType::from_le_bytes(hex!(\"{} {} {} {}    {} {} {} {}\"));",
+        c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7]);
+
+    Ok(())
+}
+
+#[derive(Debug, Clone)]
+enum Tree {
+    Branch(String, Vec<Tree>),
+    Leaf(String),
+}
+
+impl Tree {
+    fn name(&self) -> &str {
+        match self {
+            Self::Branch(name, _) => name,
+            Self::Leaf(name) => name,
+        }
+    }
+
+    fn gen_code_inner(&self, prefix: &[&str]) -> Result<()> {
+        let mut path = prefix.to_owned();
+        path.push(self.name());
+
+        match self {
+            Self::Branch(_, ref children) => {
+                for c in children.iter() {
+                    c.gen_code_inner(&path)?
+                }
+            }
+            Self::Leaf(_) => print_literal(&path)?,
+        };
+
+        Ok(())
+    }
+
+    fn gen_code(&self) -> Result<()> {
+        self.gen_code_inner(&[])
+    }
+}
+
+fn main() -> Result<()> {
+    let tree = Tree::Branch(
+        "Rosenpass IPC API".to_owned(),
+        vec![Tree::Branch(
+            "Rosenpass Protocol Server".to_owned(),
+            vec![
+                Tree::Leaf("Ping Request".to_owned()),
+                Tree::Leaf("Ping Response".to_owned()),
+                Tree::Leaf("Supply Keypair Request".to_owned()),
+                Tree::Leaf("Supply Keypair Response".to_owned()),
+                Tree::Leaf("Add Listen Socket Request".to_owned()),
+                Tree::Leaf("Add Listen Socket Response".to_owned()),
+                Tree::Leaf("Add Psk Broker Request".to_owned()),
+                Tree::Leaf("Add Psk Broker Response".to_owned()),
+            ],
+        )],
+    );
+
+    println!("type RawMsgType = u128;");
+    println!();
+    tree.gen_code()
+}

rosenpass/src/cli.rs

@@ -1,12 +1,13 @@
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Context};
 use clap::{Parser, Subcommand};
 use rosenpass_cipher_traits::Kem;
 use rosenpass_ciphers::kem::StaticKem;
 use rosenpass_secret_memory::file::StoreSecret;
-use rosenpass_util::file::{LoadValue, LoadValueB64};
+use rosenpass_util::file::{LoadValue, LoadValueB64, StoreValue};
 use rosenpass_wireguard_broker::brokers::native_unix::{
     NativeUnixBroker, NativeUnixBrokerConfigBaseBuilder, NativeUnixBrokerConfigBaseBuilderError,
 };
+use std::ops::DerefMut;
 use std::path::PathBuf;
 
 use crate::app_server::AppServerTest;
@@ -15,6 +16,29 @@ use crate::protocol::{SPk, SSk, SymKey};
 
 use super::config;
 
+#[cfg(feature = "experiment_api")]
+use {
+    command_fds::{CommandFdExt, FdMapping},
+    log::{error, info},
+    mio::net::UnixStream,
+    rosenpass_util::fd::claim_fd,
+    rosenpass_wireguard_broker::brokers::mio_client::MioBrokerClient,
+    rosenpass_wireguard_broker::WireguardBrokerMio,
+    rustix::fd::AsRawFd,
+    rustix::net::{socketpair, AddressFamily, SocketFlags, SocketType},
+    std::os::unix::net,
+    std::process::Command,
+    std::thread,
+};
+
+/// enum representing a choice of interface to a WireGuard broker
+#[derive(Debug)]
+pub enum BrokerInterface {
+    Socket(PathBuf),
+    FileDescriptor(i32),
+    SocketPair,
+}
+
 /// struct holding all CLI arguments for `clap` crate to parse
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about)]
@@ -31,11 +55,41 @@ pub struct CliArgs {
     #[arg(short, long, group = "log-level")]
     quiet: bool,
 
+    #[command(flatten)]
+    #[cfg(feature = "experiment_api")]
+    api: crate::api::cli::ApiCli,
+
+    /// path of the wireguard_psk broker socket to connect to
+    #[cfg(feature = "experiment_api")]
+    #[arg(long, group = "psk-broker-specs")]
+    psk_broker_path: Option<PathBuf>,
+
+    /// fd of the wireguard_spk broker socket to connect to
+    ///
+    /// when this command is called from another process, the other process can open and bind the
+    /// Unix socket for the psk broker connection to use themselves, passing it to this process --
+    /// in Rust this can be achieved using the
+    /// [command-fds](https://docs.rs/command-fds/latest/command_fds/) crate
+    #[cfg(feature = "experiment_api")]
+    #[arg(long, group = "psk-broker-specs")]
+    psk_broker_fd: Option<i32>,
+
+    /// spawn a psk broker locally using a socket pair
+    #[cfg(feature = "experiment_api")]
+    #[arg(short, long, group = "psk-broker-specs")]
+    psk_broker_spawn: bool,
+
     #[command(subcommand)]
     pub command: CliCommand,
 }
 
 impl CliArgs {
+    pub fn apply_to_config(&self, _cfg: &mut config::Rosenpass) -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        self.api.apply_to_config(_cfg)?;
+        Ok(())
+    }
+
     /// returns the log level filter set by CLI args
     /// returns `None` if the user did not specify any log level filter via CLI
     ///
@@ -47,13 +101,35 @@ impl CliArgs {
             return Some(log::LevelFilter::Info);
         }
         if self.quiet {
-            return Some(log::LevelFilter::Error);
+            return Some(log::LevelFilter::Warn);
         }
         if let Some(level_filter) = self.log_level {
             return Some(level_filter);
         }
         None
     }
+
+    #[cfg(feature = "experiment_api")]
+    /// returns the broker interface set by CLI args
+    /// returns `None` if the `experiment_api` feature isn't enabled
+    pub fn get_broker_interface(&self) -> Option<BrokerInterface> {
+        if let Some(path_ref) = self.psk_broker_path.as_ref() {
+            Some(BrokerInterface::Socket(path_ref.to_path_buf()))
+        } else if let Some(fd) = self.psk_broker_fd {
+            Some(BrokerInterface::FileDescriptor(fd))
+        } else if self.psk_broker_spawn {
+            Some(BrokerInterface::SocketPair)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(not(feature = "experiment_api"))]
+    /// returns the broker interface set by CLI args
+    /// returns `None` if the `experiment_api` feature isn't enabled
+    pub fn get_broker_interface(&self) -> Option<BrokerInterface> {
+        None
+    }
 }
 
 /// represents a command specified via CLI
@@ -148,14 +224,18 @@ pub enum CliCommand {
     Man,
 }
 
-impl CliCommand {
+impl CliArgs {
     /// runs the command specified via CLI
     ///
     /// ## TODO
     /// - This method consumes the [`CliCommand`] value. It might be wise to use a reference...
-    pub fn run(self, test_helpers: Option<AppServerTest>) -> anyhow::Result<()> {
+    pub fn run(
+        self,
+        broker_interface: Option<BrokerInterface>,
+        test_helpers: Option<AppServerTest>,
+    ) -> anyhow::Result<()> {
         use CliCommand::*;
-        match self {
+        match &self.command {
             Man => {
                 let man_cmd = std::process::Command::new("man")
                     .args(["1", "rosenpass"])
@@ -167,7 +247,7 @@ impl CliCommand {
             }
             GenConfig { config_file, force } => {
                 ensure!(
-                    force || !config_file.exists(),
+                    *force || !config_file.exists(),
                     "config file {config_file:?} already exists"
                 );
 
@@ -182,9 +262,9 @@ impl CliCommand {
                 let mut secret_key: Option<PathBuf> = None;
 
                 // Manual arg parsing, since clap wants to prefix flags with "--"
-                let mut args = args.into_iter();
+                let mut args = args.iter();
                 loop {
-                    match (args.next().as_deref(), args.next()) {
+                    match (args.next().map(|x| x.as_str()), args.next()) {
                         (Some("private-key"), Some(opt)) | (Some("secret-key"), Some(opt)) => {
                             secret_key = Some(opt.into());
                         }
@@ -223,10 +303,13 @@ impl CliCommand {
                         );
 
                         let config = config::Rosenpass::load(config_file)?;
+                        let keypair = config
+                            .keypair
+                            .context("Config file present, but no keypair is specified.")?;
 
-                        (config.public_key, config.secret_key)
+                        (keypair.public_key, keypair.secret_key)
                     }
-                    (_, Some(pkf), Some(skf)) => (pkf, skf),
+                    (_, Some(pkf), Some(skf)) => (pkf.clone(), skf.clone()),
                     _ => {
                         bail!("either a config-file or both public-key and secret-key file are required")
                     }
@@ -236,12 +319,14 @@ impl CliCommand {
                 let mut problems = vec![];
                 if !force && pkf.is_file() {
                     problems.push(format!(
-                        "public-key file {pkf:?} exist, refusing to overwrite it"
+                        "public-key file {:?} exists, refusing to overwrite",
+                        std::fs::canonicalize(&pkf)?,
                     ));
                 }
                 if !force && skf.is_file() {
                     problems.push(format!(
-                        "secret-key file {skf:?} exist, refusing to overwrite it"
+                        "secret-key file {:?} exists, refusing to overwrite",
+                        std::fs::canonicalize(&skf)?,
                     ));
                 }
                 if !problems.is_empty() {
@@ -258,31 +343,38 @@ impl CliCommand {
                     "config file '{config_file:?}' does not exist"
                 );
 
-                let config = config::Rosenpass::load(config_file)?;
+                let mut config = config::Rosenpass::load(config_file)?;
                 config.validate()?;
-                Self::event_loop(config, test_helpers)?;
+                self.apply_to_config(&mut config)?;
+                config.check_usefullness()?;
+
+                Self::event_loop(config, broker_interface, test_helpers)?;
             }
 
             Exchange {
                 first_arg,
-                mut rest_of_args,
+                rest_of_args,
                 config_file,
             } => {
-                rest_of_args.insert(0, first_arg);
+                let mut rest_of_args = rest_of_args.clone();
+                rest_of_args.insert(0, first_arg.clone());
                 let args = rest_of_args;
                 let mut config = config::Rosenpass::parse_args(args)?;
 
                 if let Some(p) = config_file {
-                    config.store(&p)?;
-                    config.config_file_path = p;
+                    config.store(p)?;
+                    config.config_file_path.clone_from(p);
                 }
                 config.validate()?;
-                Self::event_loop(config, test_helpers)?;
+                self.apply_to_config(&mut config)?;
+                config.check_usefullness()?;
+
+                Self::event_loop(config, broker_interface, test_helpers)?;
             }
 
             Validate { config_files } => {
                 for file in config_files {
-                    match config::Rosenpass::load(&file) {
+                    match config::Rosenpass::load(file) {
                         Ok(config) => {
                             eprintln!("{file:?} is valid TOML and conforms to the expected schema");
                             match config.validate() {
@@ -301,23 +393,34 @@ impl CliCommand {
 
     fn event_loop(
         config: config::Rosenpass,
+        broker_interface: Option<BrokerInterface>,
         test_helpers: Option<AppServerTest>,
     ) -> anyhow::Result<()> {
         const MAX_PSK_SIZE: usize = 1000;
+
         // load own keys
-        let sk = SSk::load(&config.secret_key)?;
-        let pk = SPk::load(&config.public_key)?;
+        let keypair = config
+            .keypair
+            .as_ref()
+            .map(|kp| -> anyhow::Result<_> {
+                let sk = SSk::load(&kp.secret_key)?;
+                let pk = SPk::load(&kp.public_key)?;
+                Ok((sk, pk))
+            })
+            .transpose()?;
 
         // start an application server
         let mut srv = std::boxed::Box::<AppServer>::new(AppServer::new(
-            sk,
-            pk,
-            config.listen,
+            keypair,
+            config.listen.clone(),
             config.verbosity,
             test_helpers,
         )?);
 
-        let broker_store_ptr = srv.register_broker(Box::new(NativeUnixBroker::new()))?;
+        config.apply_to_app_server(&mut srv)?;
+
+        let broker = Self::create_broker(broker_interface)?;
+        let broker_store_ptr = srv.register_broker(broker)?;
 
         fn cfg_err_map(e: NativeUnixBrokerConfigBaseBuilderError) -> anyhow::Error {
             anyhow::Error::msg(format!("NativeUnixBrokerConfigBaseBuilderError: {:?}", e))
@@ -354,13 +457,102 @@ impl CliCommand {
 
         srv.event_loop()
     }
+
+    #[cfg(feature = "experiment_api")]
+    fn create_broker(
+        broker_interface: Option<BrokerInterface>,
+    ) -> Result<
+        Box<dyn WireguardBrokerMio<MioError = anyhow::Error, Error = anyhow::Error>>,
+        anyhow::Error,
+    > {
+        if let Some(interface) = broker_interface {
+            let socket = Self::get_broker_socket(interface)?;
+            Ok(Box::new(MioBrokerClient::new(socket)))
+        } else {
+            Ok(Box::new(NativeUnixBroker::new()))
+        }
+    }
+
+    #[cfg(not(feature = "experiment_api"))]
+    fn create_broker(
+        _broker_interface: Option<BrokerInterface>,
+    ) -> Result<Box<NativeUnixBroker>, anyhow::Error> {
+        Ok(Box::new(NativeUnixBroker::new()))
+    }
+
+    #[cfg(feature = "experiment_api")]
+    fn get_broker_socket(broker_interface: BrokerInterface) -> Result<UnixStream, anyhow::Error> {
+        // Connect to the psk broker unix socket if one was specified
+        // OR OTHERWISE spawn the psk broker and use socketpair(2) to connect with them
+        match broker_interface {
+            BrokerInterface::Socket(broker_path) => Ok(UnixStream::connect(broker_path)?),
+            BrokerInterface::FileDescriptor(broker_fd) => {
+                // mio::net::UnixStream doesn't implement From<OwnedFd>, so we have to go through std
+                let sock = net::UnixStream::from(claim_fd(broker_fd)?);
+                sock.set_nonblocking(true)?;
+                Ok(UnixStream::from_std(sock))
+            }
+            BrokerInterface::SocketPair => {
+                // Form a socketpair for communicating to the broker
+                let (ours, theirs) = socketpair(
+                    AddressFamily::UNIX,
+                    SocketType::STREAM,
+                    SocketFlags::empty(),
+                    None,
+                )?;
+
+                // Setup our end of the socketpair
+                let ours = net::UnixStream::from(ours);
+                ours.set_nonblocking(true)?;
+
+                // Start the PSK broker
+                let mut child = Command::new("rosenpass-wireguard-broker-socket-handler")
+                    .args(["--stream-fd", "3"])
+                    .fd_mappings(vec![FdMapping {
+                        parent_fd: theirs.as_raw_fd(),
+                        child_fd: 3,
+                    }])?
+                    .spawn()?;
+
+                // Handle the PSK broker crashing
+                thread::spawn(move || {
+                    let status = child.wait();
+
+                    if let Ok(status) = status {
+                        if status.success() {
+                            // Maybe they are doing double forking?
+                            info!("PSK broker exited.");
+                        } else {
+                            error!("PSK broker exited with an error ({status:?})");
+                        }
+                    } else {
+                        error!("Wait on PSK broker process failed ({status:?})");
+                    }
+                });
+
+                Ok(UnixStream::from_std(ours))
+            }
+        }
+    }
 }
 
 /// generate secret and public keys, store in files according to the paths passed as arguments
 fn generate_and_save_keypair(secret_key: PathBuf, public_key: PathBuf) -> anyhow::Result<()> {
     let mut ssk = crate::protocol::SSk::random();
     let mut spk = crate::protocol::SPk::random();
-    StaticKem::keygen(ssk.secret_mut(), spk.secret_mut())?;
+    StaticKem::keygen(ssk.secret_mut(), spk.deref_mut())?;
     ssk.store_secret(secret_key)?;
     spk.store(public_key)
 }
+
+#[cfg(feature = "internal_testing")]
+pub mod testing {
+    use super::*;
+
+    pub fn generate_and_save_keypair(
+        secret_key: PathBuf,
+        public_key: PathBuf,
+    ) -> anyhow::Result<()> {
+        super::generate_and_save_keypair(secret_key, public_key)
+    }
+}

rosenpass/src/config.rs

@@ -19,13 +19,28 @@ use anyhow::{bail, ensure};
 use rosenpass_util::file::{fopen_w, Visibility};
 use serde::{Deserialize, Serialize};
 
+use crate::app_server::AppServer;
+
+#[cfg(feature = "experiment_api")]
+fn empty_api_config() -> crate::api::config::ApiConfig {
+    crate::api::config::ApiConfig {
+        listen_path: Vec::new(),
+        listen_fd: Vec::new(),
+        stream_fd: Vec::new(),
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Rosenpass {
-    /// path to the public key file
-    pub public_key: PathBuf,
+    // TODO: Raise error if secret key or public key alone is set during deserialization
+    // SEE: https://github.com/serde-rs/serde/issues/2793
+    #[serde(flatten)]
+    pub keypair: Option<Keypair>,
 
-    /// path to the secret key file
-    pub secret_key: PathBuf,
+    /// Location of the API listen sockets
+    #[cfg(feature = "experiment_api")]
+    #[serde(default = "empty_api_config")]
+    pub api: crate::api::config::ApiConfig,
 
     /// list of [`SocketAddr`] to listen on
     ///
@@ -52,9 +67,29 @@ pub struct Rosenpass {
     pub config_file_path: PathBuf,
 }
 
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)]
+pub struct Keypair {
+    /// path to the public key file
+    pub public_key: PathBuf,
+
+    /// path to the secret key file
+    pub secret_key: PathBuf,
+}
+
+impl Keypair {
+    pub fn new<Pk: AsRef<Path>, Sk: AsRef<Path>>(public_key: Pk, secret_key: Sk) -> Self {
+        let public_key = public_key.as_ref().to_path_buf();
+        let secret_key = secret_key.as_ref().to_path_buf();
+        Self {
+            public_key,
+            secret_key,
+        }
+    }
+}
+
 /// ## TODO
 /// - replace this type with [`log::LevelFilter`], also see <https://github.com/rosenpass/rosenpass/pull/246>
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Copy, Clone)]
 pub enum Verbosity {
     Quiet,
     Verbose,
@@ -107,6 +142,12 @@ pub struct WireGuard {
     pub extra_params: Vec<String>,
 }
 
+impl Default for Rosenpass {
+    fn default() -> Self {
+        Self::empty()
+    }
+}
+
 impl Rosenpass {
     /// load configuration from a TOML file
     ///
@@ -122,8 +163,10 @@ impl Rosenpass {
 
         // resolve `~` (see https://github.com/rosenpass/rosenpass/issues/237)
         use util::resolve_path_with_tilde;
-        resolve_path_with_tilde(&mut config.public_key);
-        resolve_path_with_tilde(&mut config.secret_key);
+        if let Some(ref mut keypair) = config.keypair {
+            resolve_path_with_tilde(&mut keypair.public_key);
+            resolve_path_with_tilde(&mut keypair.secret_key);
+        }
         for peer in config.peers.iter_mut() {
             resolve_path_with_tilde(&mut peer.public_key);
             if let Some(ref mut psk) = &mut peer.pre_shared_key {
@@ -157,25 +200,33 @@ impl Rosenpass {
         self.store(&self.config_file_path)
     }
 
+    pub fn apply_to_app_server(&self, _srv: &mut AppServer) -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        self.api.apply_to_app_server(_srv)?;
+        Ok(())
+    }
+
     /// Validate a configuration
     ///
     /// ## TODO
     /// - check that files do not just exist but are also readable
     /// - warn if neither out_key nor exchange_command of a peer is defined (v.i.)
     pub fn validate(&self) -> anyhow::Result<()> {
+        if let Some(ref keypair) = self.keypair {
             // check the public key file exists
             ensure!(
-            self.public_key.is_file(),
+                keypair.public_key.is_file(),
                 "could not find public-key file {:?}: no such file",
-            self.public_key
+                keypair.public_key
             );
 
             // check the secret-key file exists
             ensure!(
-            self.secret_key.is_file(),
+                keypair.secret_key.is_file(),
                 "could not find secret-key file {:?}: no such file",
-            self.secret_key
+                keypair.secret_key
             );
+        }
 
         for (i, peer) in self.peers.iter().enumerate() {
             // check peer's public-key file exists
@@ -200,12 +251,36 @@ impl Rosenpass {
         Ok(())
     }
 
+    pub fn check_usefullness(&self) -> anyhow::Result<()> {
+        #[cfg(not(feature = "experiment_api"))]
+        ensure!(self.keypair.is_some(), "Server keypair missing.");
+
+        #[cfg(feature = "experiment_api")]
+        ensure!(
+            self.keypair.is_some() || self.api.has_api_sources(),
+            "{}{}",
+            "Specify a server keypair or some API connections to configure the keypair with.",
+            "Without a keypair, rosenpass can not operate."
+        );
+
+        Ok(())
+    }
+
+    pub fn empty() -> Self {
+        Self::new(None)
+    }
+
+    pub fn from_sk_pk<Sk: AsRef<Path>, Pk: AsRef<Path>>(sk: Sk, pk: Pk) -> Self {
+        Self::new(Some(Keypair::new(pk, sk)))
+    }
+
     /// Creates a new configuration
-    pub fn new<P1: AsRef<Path>, P2: AsRef<Path>>(public_key: P1, secret_key: P2) -> Self {
+    pub fn new(keypair: Option<Keypair>) -> Self {
         Self {
-            public_key: PathBuf::from(public_key.as_ref()),
-            secret_key: PathBuf::from(secret_key.as_ref()),
+            keypair,
             listen: vec![],
+            #[cfg(feature = "experiment_api")]
+            api: crate::api::config::ApiConfig::default(),
             verbosity: Verbosity::Quiet,
             peers: vec![],
             config_file_path: PathBuf::new(),
@@ -228,7 +303,7 @@ impl Rosenpass {
     /// from chaotic args
     /// Quest: the grammar is undecideable, what do we do here?
     pub fn parse_args(args: Vec<String>) -> anyhow::Result<Self> {
-        let mut config = Self::new("", "");
+        let mut config = Self::new(Some(Keypair::new("", "")));
 
         #[derive(Debug, Hash, PartialEq, Eq)]
         enum State {
@@ -289,7 +364,7 @@ impl Rosenpass {
                         already_set.insert(OwnPublicKey),
                         "public-key was already set"
                     );
-                    config.public_key = pk.into();
+                    config.keypair.as_mut().unwrap().public_key = pk.into();
                     Own
                 }
                 (OwnSecretKey, sk, None) => {
@@ -297,7 +372,7 @@ impl Rosenpass {
                         already_set.insert(OwnSecretKey),
                         "secret-key was already set"
                     );
-                    config.secret_key = sk.into();
+                    config.keypair.as_mut().unwrap().secret_key = sk.into();
                     Own
                 }
                 (OwnListen, l, None) => {
@@ -432,10 +507,12 @@ impl Rosenpass {
         };
 
         Self {
+            keypair: Some(Keypair {
                 public_key: "/path/to/rp-public-key".into(),
                 secret_key: "/path/to/rp-secret-key".into(),
+            }),
             peers: vec![peer],
-            ..Self::new("", "")
+            ..Self::new(None)
         }
     }
 }
@@ -448,13 +525,119 @@ impl Default for Verbosity {
 
 #[cfg(test)]
 mod test {
+
     use super::*;
-    use std::net::IpAddr;
+    use std::{borrow::Borrow, net::IpAddr};
+
+    fn toml_des<S: Borrow<str>>(s: S) -> Result<toml::Table, toml::de::Error> {
+        toml::from_str(s.borrow())
+    }
+
+    fn toml_ser<S: Serialize>(s: S) -> Result<toml::Table, toml::ser::Error> {
+        toml::Table::try_from(s)
+    }
+
+    fn assert_toml<L: Serialize, R: Borrow<str>>(l: L, r: R, info: &str) -> anyhow::Result<()> {
+        fn lines_prepend(prefix: &str, s: &str) -> anyhow::Result<String> {
+            use std::fmt::Write;
+
+            let mut buf = String::new();
+            for line in s.lines() {
+                writeln!(&mut buf, "{prefix}{line}")?;
+            }
+            Ok(buf)
+        }
+
+        let l = toml_ser(l)?;
+        let r = toml_des(r.borrow())?;
+        ensure!(
+            l == r,
+            "{}{}TOML value mismatch.\n  Have:\n{}\n  Expected:\n{}",
+            info,
+            if info.is_empty() { "" } else { ": " },
+            lines_prepend("    ", &toml::to_string_pretty(&l)?)?,
+            lines_prepend("    ", &toml::to_string_pretty(&r)?)?
+        );
+        Ok(())
+    }
+
+    fn assert_toml_round<'de, L: Serialize + Deserialize<'de>, R: Borrow<str>>(
+        l: L,
+        r: R,
+    ) -> anyhow::Result<()> {
+        let l = toml_ser(l)?;
+        assert_toml(&l, r.borrow(), "Straight deserialization")?;
+
+        let l: L = l.try_into().unwrap();
+        let l = toml_ser(l).unwrap();
+        assert_toml(l, r.borrow(), "Roundtrip deserialization")?;
+
+        Ok(())
+    }
 
     fn split_str(s: &str) -> Vec<String> {
         s.split(' ').map(|s| s.to_string()).collect()
     }
 
+    #[test]
+    fn toml_serialization() -> anyhow::Result<()> {
+        #[cfg(feature = "experiment_api")]
+        assert_toml_round(
+            Rosenpass::empty(),
+            r#"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+
+            [api]
+            listen_path = []
+            listen_fd = []
+            stream_fd = []
+        "#,
+        )?;
+
+        #[cfg(not(feature = "experiment_api"))]
+        assert_toml_round(
+            Rosenpass::empty(),
+            r#"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+        "#,
+        )?;
+
+        #[cfg(feature = "experiment_api")]
+        assert_toml_round(
+            Rosenpass::from_sk_pk("/my/sk", "/my/pk"),
+            r#"
+            public_key = "/my/pk"
+            secret_key = "/my/sk"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+
+            [api]
+            listen_path = []
+            listen_fd = []
+            stream_fd = []
+        "#,
+        )?;
+
+        #[cfg(not(feature = "experiment_api"))]
+        assert_toml_round(
+            Rosenpass::from_sk_pk("/my/sk", "/my/pk"),
+            r#"
+            public_key = "/my/pk"
+            secret_key = "/my/sk"
+            listen = []
+            verbosity = "Quiet"
+            peers = []
+        "#,
+        )?;
+
+        Ok(())
+    }
+
     #[test]
     fn test_simple_cli_parse() {
         let args = split_str(
@@ -465,8 +648,10 @@ mod test {
 
         let config = Rosenpass::parse_args(args).unwrap();
 
-        assert_eq!(config.public_key, PathBuf::from("/my/public-key"));
-        assert_eq!(config.secret_key, PathBuf::from("/my/secret-key"));
+        assert_eq!(
+            config.keypair,
+            Some(Keypair::new("/my/public-key", "/my/secret-key"))
+        );
         assert_eq!(config.verbosity, Verbosity::Verbose);
         assert_eq!(
             &config.listen,
@@ -495,8 +680,10 @@ mod test {
 
         let config = Rosenpass::parse_args(args).unwrap();
 
-        assert_eq!(config.public_key, PathBuf::from("/my/public-key"));
-        assert_eq!(config.secret_key, PathBuf::from("/my/secret-key"));
+        assert_eq!(
+            config.keypair,
+            Some(Keypair::new("/my/public-key", "/my/secret-key"))
+        );
         assert_eq!(config.verbosity, Verbosity::Verbose);
         assert!(&config.listen.is_empty());
         assert_eq!(

rosenpass/src/lib.rs

@@ -1,3 +1,5 @@
+#[cfg(feature = "experiment_api")]
+pub mod api;
 pub mod app_server;
 pub mod cli;
 pub mod config;
@@ -11,4 +13,8 @@ pub enum RosenpassError {
     BufferSizeMismatch,
     #[error("invalid message type")]
     InvalidMessageType(u8),
+    #[error("invalid API message type")]
+    InvalidApiMessageType(u128),
+    #[error("could not parse API message")]
+    InvalidApiMessage,
 }

rosenpass/src/main.rs

@@ -8,6 +8,14 @@ pub fn main() {
     // parse CLI arguments
     let args = CliArgs::parse();
 
+    {
+        use rosenpass_secret_memory as SM;
+        #[cfg(feature = "experiment_memfd_secret")]
+        SM::secret_policy_try_use_memfd_secrets();
+        #[cfg(not(feature = "experiment_memfd_secret"))]
+        SM::secret_policy_use_only_malloc_secrets();
+    }
+
     // init logging
     {
         let mut log_builder = env_logger::Builder::from_default_env(); // sets log level filter from environment (or defaults)
@@ -26,10 +34,11 @@ pub fn main() {
         // error!("error dummy");
     }
 
-    match args.command.run(None) {
+    let broker_interface = args.get_broker_interface();
+    match args.run(broker_interface, None) {
         Ok(_) => {}
         Err(e) => {
-            error!("{e}");
+            error!("{e:?}");
             exit(1);
         }
     }

rosenpass/src/protocol/build_crypto_server.rs

@@ -0,0 +1,127 @@
+use rosenpass_util::{
+    build::Build,
+    mem::{DiscardResultExt, SwapWithDefaultExt},
+    result::ensure_or,
+};
+use thiserror::Error;
+
+use super::{CryptoServer, PeerPtr, SPk, SSk, SymKey};
+
+#[derive(Debug, Clone)]
+pub struct Keypair {
+    pub sk: SSk,
+    pub pk: SPk,
+}
+
+// TODO: We need a named tuple derive
+impl Keypair {
+    pub fn new(sk: SSk, pk: SPk) -> Self {
+        Self { sk, pk }
+    }
+
+    pub fn zero() -> Self {
+        Self::new(SSk::zero(), SPk::zero())
+    }
+
+    pub fn random() -> Self {
+        Self::new(SSk::random(), SPk::random())
+    }
+
+    pub fn from_parts(parts: (SSk, SPk)) -> Self {
+        Self::new(parts.0, parts.1)
+    }
+
+    pub fn into_parts(self) -> (SSk, SPk) {
+        (self.sk, self.pk)
+    }
+}
+
+#[derive(Error, Debug)]
+#[error("PSK already set in BuildCryptoServer")]
+pub struct PskAlreadySet;
+
+#[derive(Error, Debug)]
+#[error("Keypair already set in BuildCryptoServer")]
+pub struct KeypairAlreadySet;
+
+#[derive(Error, Debug)]
+#[error("Can not construct CryptoServer: Missing keypair")]
+pub struct MissingKeypair;
+
+#[derive(Debug, Default)]
+pub struct BuildCryptoServer {
+    pub keypair: Option<Keypair>,
+    pub peers: Vec<PeerParams>,
+}
+
+impl Build<CryptoServer> for BuildCryptoServer {
+    type Error = anyhow::Error;
+
+    fn build(self) -> Result<CryptoServer, Self::Error> {
+        let Some(Keypair { sk, pk }) = self.keypair else {
+            return Err(MissingKeypair)?;
+        };
+
+        let mut srv = CryptoServer::new(sk, pk);
+
+        for (idx, PeerParams { psk, pk }) in self.peers.into_iter().enumerate() {
+            let PeerPtr(idx2) = srv.add_peer(psk, pk)?;
+            assert!(idx == idx2, "Peer id changed during CryptoServer construction from {idx} to {idx2}. This is a developer error.")
+        }
+
+        Ok(srv)
+    }
+}
+
+#[derive(Debug)]
+pub struct PeerParams {
+    pub psk: Option<SymKey>,
+    pub pk: SPk,
+}
+
+impl BuildCryptoServer {
+    pub fn new(keypair: Option<Keypair>, peers: Vec<PeerParams>) -> Self {
+        Self { keypair, peers }
+    }
+
+    pub fn empty() -> Self {
+        Self::new(None, Vec::new())
+    }
+
+    pub fn from_parts(parts: (Option<Keypair>, Vec<PeerParams>)) -> Self {
+        Self {
+            keypair: parts.0,
+            peers: parts.1,
+        }
+    }
+
+    pub fn take_parts(&mut self) -> (Option<Keypair>, Vec<PeerParams>) {
+        (self.keypair.take(), self.peers.swap_with_default())
+    }
+
+    pub fn into_parts(mut self) -> (Option<Keypair>, Vec<PeerParams>) {
+        self.take_parts()
+    }
+
+    pub fn with_keypair(&mut self, keypair: Keypair) -> Result<&mut Self, KeypairAlreadySet> {
+        ensure_or(self.keypair.is_none(), KeypairAlreadySet)?;
+        self.keypair.insert(keypair).discard_result();
+        Ok(self)
+    }
+
+    pub fn with_added_peer(&mut self, psk: Option<SymKey>, pk: SPk) -> &mut Self {
+        // TODO: Check here already whether peer was already added
+        self.peers.push(PeerParams { psk, pk });
+        self
+    }
+
+    pub fn add_peer(&mut self, psk: Option<SymKey>, pk: SPk) -> PeerPtr {
+        let id = PeerPtr(self.peers.len());
+        self.with_added_peer(psk, pk);
+        id
+    }
+
+    pub fn emancipate(&mut self) -> Self {
+        Self::from_parts(self.take_parts())
+    }
+}

rosenpass/src/protocol/mod.rs

@@ -0,0 +1,6 @@
+mod build_crypto_server;
+#[allow(clippy::module_inception)]
+mod protocol;
+
+pub use build_crypto_server::*;
+pub use protocol::*;

rosenpass/src/protocol/protocol.rs

@@ -19,20 +19,25 @@
 //! [CryptoServer].
 //!
 //! ```
+//! use std::ops::DerefMut;
+//! use rosenpass_secret_memory::policy::*;
 //! use rosenpass_cipher_traits::Kem;
 //! use rosenpass_ciphers::kem::StaticKem;
 //! use rosenpass::{
 //!     protocol::{SSk, SPk, MsgBuf, PeerPtr, CryptoServer, SymKey},
 //! };
 //! # fn main() -> anyhow::Result<()> {
+//! // Set security policy for storing secrets
+//!
+//! secret_policy_try_use_memfd_secrets();
 //!
 //! // initialize secret and public key for peer a ...
 //! let (mut peer_a_sk, mut peer_a_pk) = (SSk::zero(), SPk::zero());
-//! StaticKem::keygen(peer_a_sk.secret_mut(), peer_a_pk.secret_mut())?;
+//! StaticKem::keygen(peer_a_sk.secret_mut(), peer_a_pk.deref_mut())?;
 //!
 //! // ... and for peer b
 //! let (mut peer_b_sk, mut peer_b_pk) = (SSk::zero(), SPk::zero());
-//! StaticKem::keygen(peer_b_sk.secret_mut(), peer_b_pk.secret_mut())?;
+//! StaticKem::keygen(peer_b_sk.secret_mut(), peer_b_pk.deref_mut())?;
 //!
 //! // initialize server and a pre-shared key
 //! let psk = SymKey::random();
@@ -67,6 +72,7 @@
 
 use std::convert::Infallible;
 use std::mem::size_of;
+use std::ops::Deref;
 use std::{
     collections::hash_map::{
         Entry::{Occupied, Vacant},
@@ -84,20 +90,14 @@ use rosenpass_ciphers::hash_domain::{SecretHashDomain, SecretHashDomainNamespace
 use rosenpass_ciphers::kem::{EphemeralKem, StaticKem};
 use rosenpass_ciphers::{aead, xaead, KEY_LEN};
 use rosenpass_constant_time as constant_time;
-use rosenpass_secret_memory::{Public, Secret};
-use rosenpass_util::{cat, mem::cpy_min, ord::max_usize, time::Timebase};
+use rosenpass_secret_memory::{Public, PublicBox, Secret};
+use rosenpass_util::{cat, mem::cpy_min, time::Timebase};
 use zerocopy::{AsBytes, FromBytes, Ref};
 
 use crate::{hash_domains, msgs::*, RosenpassError};
 
 // CONSTANTS & SETTINGS //////////////////////////
 
-/// Size required to fit any message in binary form
-pub const RTX_BUFFER_SIZE: usize = max_usize(
-    size_of::<Envelope<InitHello>>(),
-    size_of::<Envelope<InitConf>>(),
-);
-
 /// A type for time, e.g. for backoff before re-tries
 pub type Timing = f64;
 
@@ -159,7 +159,7 @@ pub fn has_happened(ev: Timing, now: Timing) -> bool {
 
 // DATA STRUCTURES & BASIC TRAITS & ACCESSORS ////
 
-pub type SPk = Secret<{ StaticKem::PK_LEN }>; // Just Secret<> instead of Public<> so it gets allocated on the heap
+pub type SPk = PublicBox<{ StaticKem::PK_LEN }>;
 pub type SSk = Secret<{ StaticKem::SK_LEN }>;
 pub type EPk = Public<{ EphemeralKem::PK_LEN }>;
 pub type ESk = Secret<{ EphemeralKem::SK_LEN }>;
@@ -544,7 +544,7 @@ impl CryptoServer {
     pub fn pidm(&self) -> Result<PeerId> {
         Ok(Public::new(
             hash_domains::peerid()?
-                .mix(self.spkm.secret())?
+                .mix(self.spkm.deref())?
                 .into_value()))
     }
 
@@ -704,7 +704,7 @@ impl Peer {
     pub fn pidt(&self) -> Result<PeerId> {
         Ok(Public::new(
             hash_domains::peerid()?
-                .mix(self.spkt.secret())?
+                .mix(self.spkt.deref())?
                 .into_value()))
     }
 }
@@ -1013,7 +1013,7 @@ impl CryptoServer {
 
         let cookie_value = active_cookie_value.unwrap();
         let cookie_key = hash_domains::cookie_key()?
-            .mix(self.spkm.secret())?
+            .mix(self.spkm.deref())?
             .into_value();
 
         let mut msg_out = truncating_cast_into::<CookieReply>(tx_buf)?;
@@ -1505,7 +1505,7 @@ where
     /// Calculate the message authentication code (`mac`) and also append cookie value
     pub fn seal(&mut self, peer: PeerPtr, srv: &CryptoServer) -> Result<()> {
         let mac = hash_domains::mac()?
-            .mix(peer.get(srv).spkt.secret())?
+            .mix(peer.get(srv).spkt.deref())?
             .mix(&self.as_bytes()[span_of!(Self, msg_type..mac)])?;
         self.mac.copy_from_slice(mac.into_value()[..16].as_ref());
         self.seal_cookie(peer, srv)?;
@@ -1532,7 +1532,7 @@ where
     /// Check the message authentication code
     pub fn check_seal(&self, srv: &CryptoServer) -> Result<bool> {
         let expected = hash_domains::mac()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(&self.as_bytes()[span_of!(Self, msg_type..mac)])?;
         Ok(constant_time::memcmp(
             &self.mac,
@@ -1637,7 +1637,7 @@ impl HandshakeState {
 
         // calculate ad contents
         let ad = hash_domains::biscuit_ad()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(self.sidi.as_slice())?
             .mix(self.sidr.as_slice())?
             .into_value();
@@ -1672,7 +1672,7 @@ impl HandshakeState {
 
         // Calculate additional data fields
         let ad = hash_domains::biscuit_ad()?
-            .mix(srv.spkm.secret())?
+            .mix(srv.spkm.deref())?
             .mix(sidi.as_slice())?
             .mix(sidr.as_slice())?
             .into_value();
@@ -1759,7 +1759,7 @@ impl CryptoServer {
         let mut hs = InitiatorHandshake::zero_with_timestamp(self);
 
         // IHI1
-        hs.core.init(peer.get(self).spkt.secret())?;
+        hs.core.init(peer.get(self).spkt.deref())?;
 
         // IHI2
         hs.core.sidi.randomize();
@@ -1776,7 +1776,7 @@ impl CryptoServer {
         hs.core
             .encaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
                 ih.sctr.as_mut_slice(),
-                peer.get(self).spkt.secret(),
+                peer.get(self).spkt.deref(),
             )?;
 
         // IHI6
@@ -1785,7 +1785,7 @@ impl CryptoServer {
 
         // IHI7
         hs.core
-            .mix(self.spkm.secret())?
+            .mix(self.spkm.deref())?
             .mix(peer.get(self).psk.secret())?;
 
         // IHI8
@@ -1803,7 +1803,7 @@ impl CryptoServer {
         core.sidi = SessionId::from_slice(&ih.sidi);
 
         // IHR1
-        core.init(self.spkm.secret())?;
+        core.init(self.spkm.deref())?;
 
         // IHR4
         core.mix(&ih.sidi)?.mix(&ih.epki)?;
@@ -1811,7 +1811,7 @@ impl CryptoServer {
         // IHR5
         core.decaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             self.sskm.secret(),
-            self.spkm.secret(),
+            self.spkm.deref(),
             &ih.sctr,
         )?;
 
@@ -1824,7 +1824,7 @@ impl CryptoServer {
         };
 
         // IHR7
-        core.mix(peer.get(self).spkt.secret())?
+        core.mix(peer.get(self).spkt.deref())?
             .mix(peer.get(self).psk.secret())?;
 
         // IHR8
@@ -1844,7 +1844,7 @@ impl CryptoServer {
         // RHR5
         core.encaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             &mut rh.scti,
-            peer.get(self).spkt.secret(),
+            peer.get(self).spkt.deref(),
         )?;
 
         // RHR6
@@ -1905,14 +1905,14 @@ impl CryptoServer {
         // RHI4
         core.decaps_and_mix::<EphemeralKem, { EphemeralKem::SHK_LEN }>(
             hs!().eski.secret(),
-            &*hs!().epki,
+            hs!().epki.deref(),
             &rh.ecti,
         )?;
 
         // RHI5
         core.decaps_and_mix::<StaticKem, { StaticKem::SHK_LEN }>(
             self.sskm.secret(),
-            self.spkm.secret(),
+            self.spkm.deref(),
             &rh.scti,
         )?;
 
@@ -2010,8 +2010,7 @@ impl CryptoServer {
 
         // Send ack – Implementing sending the empty acknowledgement here
         // instead of a generic PeerPtr::send(&Server, Option<&[u8]>) -> Either<EmptyData, Data>
-        // because data transmission is a stub currently. This software is supposed to be used
-        // as a key exchange service feeding a PSK into some classical (i.e. non post quantum)
+        // because data transmission is a stub currently.
         let ses = peer
             .session()
             .get_mut(self)
@@ -2109,7 +2108,7 @@ impl CryptoServer {
                     ),
                 }?;
 
-                let spkt = peer.get(self).spkt.secret();
+                let spkt = peer.get(self).spkt.deref();
                 let cookie_key = hash_domains::cookie_key()?.mix(spkt)?.into_value();
                 let cookie_value = peer.cv().update_mut(self).unwrap();
 
@@ -2142,9 +2141,10 @@ fn truncating_cast_into_nomut<T: FromBytes>(buf: &[u8]) -> Result<Ref<&[u8], T>,
 
 #[cfg(test)]
 mod test {
-    use std::{net::SocketAddrV4, thread::sleep, time::Duration};
+    use std::{net::SocketAddrV4, ops::DerefMut, thread::sleep, time::Duration};
 
     use super::*;
+    use serial_test::serial;
 
     struct VecHostIdentifier(Vec<u8>);
 
@@ -2166,7 +2166,21 @@ mod test {
         }
     }
 
+    fn setup_logging() {
+        use std::io::Write;
+        let mut log_builder = env_logger::Builder::from_default_env(); // sets log level filter from environment (or defaults)
+        log_builder.filter_level(log::LevelFilter::Info);
+        log_builder.format_timestamp_nanos();
+        log_builder.format(|buf, record| {
+            let ts_format = buf.timestamp_nanos().to_string();
+            writeln!(buf, "{}: {}", &ts_format[14..], record.args())
+        });
+
+        let _ = log_builder.try_init();
+    }
+
     #[test]
+    #[serial]
     /// Ensure that the protocol implementation can deal with truncated
     /// messages and with overlong messages.
     ///
@@ -2182,6 +2196,8 @@ mod test {
     /// Through all this, the handshake should still successfully terminate;
     /// i.e. an exchanged key must be produced in both servers.
     fn handles_incorrect_size_messages() {
+        setup_logging();
+        rosenpass_secret_memory::secret_policy_try_use_memfd_secrets();
         stacker::grow(8 * 1024 * 1024, || {
             const OVERSIZED_MESSAGE: usize = ((MAX_MESSAGE_LEN as f32) * 1.2) as usize;
             type MsgBufPlus = Public<OVERSIZED_MESSAGE>;
@@ -2234,7 +2250,7 @@ mod test {
     fn keygen() -> Result<(SSk, SPk)> {
         // TODO: Copied from the benchmark; deduplicate
         let (mut sk, mut pk) = (SSk::zero(), SPk::zero());
-        StaticKem::keygen(sk.secret_mut(), pk.secret_mut())?;
+        StaticKem::keygen(sk.secret_mut(), pk.deref_mut())?;
         Ok((sk, pk))
     }
 
@@ -2252,7 +2268,10 @@ mod test {
     }
 
     #[test]
+    #[serial]
     fn test_regular_exchange() {
+        setup_logging();
+        rosenpass_secret_memory::secret_policy_try_use_memfd_secrets();
         stacker::grow(8 * 1024 * 1024, || {
             type MsgBufPlus = Public<MAX_MESSAGE_LEN>;
             let (mut a, mut b) = make_server_pair().unwrap();
@@ -2296,7 +2315,7 @@ mod test {
 
             //B handles InitConf, sends EmptyData
             let HandleMsgResult {
-                resp,
+                resp: _,
                 exchanged_with,
             } = b
                 .handle_msg(&a_to_b_buf.as_slice()[..init_conf_len], &mut *b_to_a_buf)
@@ -2310,7 +2329,10 @@ mod test {
     }
 
     #[test]
+    #[serial]
     fn test_regular_init_conf_retransmit() {
+        setup_logging();
+        rosenpass_secret_memory::secret_policy_try_use_memfd_secrets();
         stacker::grow(8 * 1024 * 1024, || {
             type MsgBufPlus = Public<MAX_MESSAGE_LEN>;
             let (mut a, mut b) = make_server_pair().unwrap();
@@ -2355,7 +2377,7 @@ mod test {
 
             //B handles InitConf, sends EmptyData
             let HandleMsgResult {
-                resp,
+                resp: _,
                 exchanged_with,
             } = b
                 .handle_msg(&a_to_b_buf.as_slice()[..init_conf_len], &mut *b_to_a_buf)
@@ -2368,7 +2390,7 @@ mod test {
 
             //B handles InitConf again, sends EmptyData
             let HandleMsgResult {
-                resp,
+                resp: _,
                 exchanged_with,
             } = b
                 .handle_msg(&a_to_b_buf.as_slice()[..init_conf_len], &mut *b_to_a_buf)
@@ -2382,7 +2404,10 @@ mod test {
     }
 
     #[test]
+    #[serial]
     fn cookie_reply_mechanism_responder_under_load() {
+        setup_logging();
+        rosenpass_secret_memory::secret_policy_try_use_memfd_secrets();
         stacker::grow(8 * 1024 * 1024, || {
             type MsgBufPlus = Public<MAX_MESSAGE_LEN>;
             let (mut a, mut b) = make_server_pair().unwrap();
@@ -2476,7 +2501,10 @@ mod test {
     }
 
     #[test]
+    #[serial]
     fn cookie_reply_mechanism_initiator_bails_on_message_under_load() {
+        setup_logging();
+        rosenpass_secret_memory::secret_policy_try_use_memfd_secrets();
         stacker::grow(8 * 1024 * 1024, || {
             type MsgBufPlus = Public<MAX_MESSAGE_LEN>;
             let (mut a, mut b) = make_server_pair().unwrap();

rosenpass/tests/api-integration-tests-api-setup.rs

@@ -0,0 +1,332 @@
+use std::{
+    borrow::Borrow,
+    io::{BufRead, BufReader, Write},
+    os::unix::net::UnixStream,
+    process::Stdio,
+    thread::sleep,
+    time::Duration,
+};
+
+use anyhow::{bail, Context};
+use command_fds::{CommandFdExt, FdMapping};
+use hex_literal::hex;
+use rosenpass::api::{
+    self, add_listen_socket_response_status, add_psk_broker_response_status,
+    supply_keypair_response_status,
+};
+use rosenpass_util::{
+    b64::B64Display,
+    file::LoadValueB64,
+    io::IoErrorKind,
+    length_prefix_encoding::{decoder::LengthPrefixDecoder, encoder::LengthPrefixEncoder},
+    mem::{DiscardResultExt, MoveExt},
+    mio::WriteWithFileDescriptors,
+    zerocopy::ZerocopySliceExt,
+};
+use rustix::fd::{AsFd, AsRawFd};
+use tempfile::TempDir;
+use zerocopy::AsBytes;
+
+use rosenpass::protocol::SymKey;
+
+struct KillChild(std::process::Child);
+
+impl Drop for KillChild {
+    fn drop(&mut self) {
+        self.0.kill().discard_result();
+        self.0.wait().discard_result()
+    }
+}
+
+#[test]
+fn api_integration_api_setup() -> anyhow::Result<()> {
+    rosenpass_secret_memory::policy::secret_policy_use_only_malloc_secrets();
+
+    let dir = TempDir::with_prefix("rosenpass-api-integration-test")?;
+
+    macro_rules! tempfile {
+        ($($lst:expr),+) => {{
+            let mut buf =  dir.path().to_path_buf();
+            $(buf.push($lst);)*
+            buf
+        }}
+    }
+
+    let peer_a_endpoint = "[::1]:0";
+    let peer_a_listen = std::net::UdpSocket::bind(peer_a_endpoint)?;
+    let peer_a_endpoint = format!("{}", peer_a_listen.local_addr()?);
+    let peer_a_keypair = config::Keypair::new(tempfile!("a.pk"), tempfile!("a.sk"));
+
+    let peer_b_osk = tempfile!("b.osk");
+    let peer_b_wg_device = "mock_device";
+    let peer_b_wg_peer_id = hex!(
+        "
+        93 0f ee 77 0c 6b 54 7e  13 5f 13 92 21 97 26 53
+        7d 77 4a 6a 0f 6c eb 1a  dd 6e 5b c4 1b 92 cd 99
+    "
+    );
+
+    use rosenpass::config;
+    let peer_a = config::Rosenpass {
+        config_file_path: tempfile!("a.config"),
+        keypair: None,
+        listen: vec![], // TODO: This could collide by accident
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("a.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("b.pk"),
+            key_out: None,
+            endpoint: None,
+            pre_shared_key: None,
+            wg: Some(config::WireGuard {
+                device: peer_b_wg_device.to_string(),
+                peer: format!("{}", peer_b_wg_peer_id.fmt_b64::<8129>()),
+                extra_params: vec![],
+            }),
+        }],
+    };
+
+    let peer_b_keypair = config::Keypair::new(tempfile!("b.pk"), tempfile!("b.sk"));
+    let peer_b = config::Rosenpass {
+        config_file_path: tempfile!("b.config"),
+        keypair: Some(peer_b_keypair.clone()),
+        listen: vec![],
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("b.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("a.pk"),
+            key_out: Some(peer_b_osk.clone()),
+            endpoint: Some(peer_a_endpoint.to_owned()),
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    // Generate the keys
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_a_keypair.secret_key.clone(),
+        peer_a_keypair.public_key.clone(),
+    )?;
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_b_keypair.secret_key.clone(),
+        peer_b_keypair.public_key.clone(),
+    )?;
+
+    // Write the configuration files
+    peer_a.commit()?;
+    peer_b.commit()?;
+
+    let (deliberate_fail_api_client, deliberate_fail_api_server) =
+        std::os::unix::net::UnixStream::pair()?;
+    let deliberate_fail_child_fd = 3;
+
+    // Start peer a
+    let _proc_a = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args(["--api-stream-fd", &deliberate_fail_child_fd.to_string()])
+            .fd_mappings(vec![FdMapping {
+                parent_fd: deliberate_fail_api_server.move_here().as_raw_fd(),
+                child_fd: 3,
+            }])?
+            .args([
+                "exchange-config",
+                peer_a.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?,
+    );
+
+    // Start peer b
+    let mut proc_b = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_b.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stderr(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Acquire stdout
+    let mut out_b = BufReader::new(proc_b.0.stdout.take().context("")?).lines();
+
+    // Now connect to the peers
+    let api_path = peer_a.api.listen_path[0].as_path();
+
+    // Wait for the socket to be created
+    let attempt = 0;
+    while !api_path.exists() {
+        sleep(Duration::from_millis(200));
+        assert!(
+            attempt < 50,
+            "Api failed to be created even after 50 seconds"
+        );
+    }
+
+    let api = UnixStream::connect(api_path)?;
+    let (psk_broker_sock, psk_broker_server_sock) = UnixStream::pair()?;
+
+    // Send AddListenSocket request
+    {
+        let fd = peer_a_listen.as_fd();
+
+        let mut fds = vec![&fd].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::AddListenSocketRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+        std::mem::forget(peer_a_listen);
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::AddListenSocketResponse>()?;
+        assert_eq!(
+            *res,
+            api::AddListenSocketResponse::new(add_listen_socket_response_status::OK)
+        );
+    }
+
+    // Deliberately break API connection given via FD; this checks that the
+    // API connections are closed when invalid data is received and it also
+    // implicitly checks that other connections are unaffected
+    {
+        use std::io::ErrorKind as K;
+        let client = deliberate_fail_api_client;
+        let err = loop {
+            if let Err(e) = client.borrow().write(&[0xffu8; 16]) {
+                break e;
+            }
+        };
+        // NotConnected happens on Mac
+        assert!(matches!(
+            err.io_error_kind(),
+            K::ConnectionReset | K::BrokenPipe | K::NotConnected
+        ));
+    }
+
+    // Send SupplyKeypairRequest
+    {
+        use rustix::fs::{open, Mode, OFlags};
+        let sk = open(peer_a_keypair.secret_key, OFlags::RDONLY, Mode::empty())?;
+        let pk = open(peer_a_keypair.public_key, OFlags::RDONLY, Mode::empty())?;
+
+        let mut fds = vec![&sk, &pk].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::SupplyKeypairRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::SupplyKeypairResponse>()?;
+        assert_eq!(
+            *res,
+            api::SupplyKeypairResponse::new(supply_keypair_response_status::OK)
+        );
+    }
+
+    // Send AddPskBroker request
+    {
+        let mut fds = vec![psk_broker_server_sock.as_fd()].into();
+        let mut api = WriteWithFileDescriptors::<UnixStream, _, _, _>::new(&api, &mut fds);
+        LengthPrefixEncoder::from_message(api::AddPskBrokerRequest::new().as_bytes())
+            .write_all_to_stdio(&mut api)?;
+        assert!(fds.is_empty(), "Failed to write all file descriptors");
+    }
+
+    // Read response
+    {
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(&api)?;
+        let res = res.zk_parse::<api::AddPskBrokerResponse>()?;
+        assert_eq!(
+            *res,
+            api::AddPskBrokerResponse::new(add_psk_broker_response_status::OK)
+        );
+    }
+
+    // Wait for the keys to successfully exchange a key
+    let mut attempt = 0;
+    loop {
+        // Read OSK generated by A
+        let osk_a = {
+            use rosenpass_wireguard_broker::api::msgs as M;
+            type SetPskReqPkg = M::Envelope<M::SetPskRequest>;
+            type SetPskResPkg = M::Envelope<M::SetPskResponse>;
+
+            // Receive request
+            let mut decoder = LengthPrefixDecoder::new([0u8; M::REQUEST_MSG_BUFFER_SIZE]);
+            let req = decoder.read_all_from_stdio(&psk_broker_sock)?;
+
+            let req = req.zk_parse::<SetPskReqPkg>()?;
+            assert_eq!(req.msg_type, M::MsgType::SetPsk as u8);
+            assert_eq!(req.payload.peer_id, peer_b_wg_peer_id);
+            assert_eq!(req.payload.iface()?, peer_b_wg_device);
+
+            // Send response
+            let res = SetPskResPkg {
+                msg_type: M::MsgType::SetPsk as u8,
+                reserved: [0u8; 3],
+                payload: M::SetPskResponse {
+                    return_code: M::SetPskResponseReturnCode::Success as u8,
+                },
+            };
+            LengthPrefixEncoder::from_message(res.as_bytes())
+                .write_all_to_stdio(&psk_broker_sock)?;
+
+            SymKey::from_slice(&req.payload.psk)
+        };
+
+        // Read OSK generated by B
+        let osk_b = {
+            let line = out_b.next().context("")??;
+            let words = line.split(' ').collect::<Vec<_>>();
+
+            // FIXED     FIXED PEER-ID                                      FIXED    FILENAME       STATUS
+            // output-key peer KZqXTZ4l2aNnkJtLPhs4D8JxHTGmRSL9w3Qr+X8JxFk= key-file "client-A-osk" exchanged
+            let peer_id = words
+                .get(2)
+                .with_context(|| format!("Bad rosenpass output: `{line}`"))?;
+            assert_eq!(
+                line,
+                format!(
+                    "output-key peer {peer_id} key-file \"{}\" exchanged",
+                    peer_b_osk.to_str().context("")?
+                )
+            );
+
+            SymKey::load_b64::<64, _>(peer_b_osk.clone())?
+        };
+
+        // TODO: This may be flaky. Both rosenpass instances are not guaranteed to produce
+        // the same number of output events; they merely guarantee eventual consistency of OSK.
+        // Correctly, we should use tokio to read any number of generated OSKs and indicate
+        // success on consensus
+        match osk_a.secret() == osk_b.secret() {
+            true => break,
+            false if attempt > 10 => bail!("Peers did not produce a matching key even after ten attempts. Something is wrong with the key exchange!"),
+            false => {},
+        };
+
+        attempt += 1;
+    }
+
+    Ok(())
+}

rosenpass/tests/api-integration-tests.rs

@@ -0,0 +1,194 @@
+use std::{
+    io::{BufRead, BufReader},
+    net::ToSocketAddrs,
+    os::unix::net::UnixStream,
+    process::Stdio,
+};
+
+use anyhow::{bail, Context};
+use rosenpass::api;
+use rosenpass_to::{ops::copy_slice_least_src, To};
+use rosenpass_util::{
+    file::LoadValueB64,
+    length_prefix_encoding::{decoder::LengthPrefixDecoder, encoder::LengthPrefixEncoder},
+};
+use rosenpass_util::{mem::DiscardResultExt, zerocopy::ZerocopySliceExt};
+use tempfile::TempDir;
+use zerocopy::AsBytes;
+
+use rosenpass::protocol::SymKey;
+
+struct KillChild(std::process::Child);
+
+impl Drop for KillChild {
+    fn drop(&mut self) {
+        self.0.kill().discard_result();
+        self.0.wait().discard_result()
+    }
+}
+
+#[test]
+fn api_integration_test() -> anyhow::Result<()> {
+    rosenpass_secret_memory::policy::secret_policy_use_only_malloc_secrets();
+
+    let dir = TempDir::with_prefix("rosenpass-api-integration-test")?;
+
+    macro_rules! tempfile {
+        ($($lst:expr),+) => {{
+            let mut buf =  dir.path().to_path_buf();
+            $(buf.push($lst);)*
+            buf
+        }}
+    }
+
+    let peer_a_endpoint = "[::1]:61423";
+    let peer_a_osk = tempfile!("a.osk");
+    let peer_b_osk = tempfile!("b.osk");
+
+    use rosenpass::config;
+
+    let peer_a_keypair = config::Keypair::new(tempfile!("a.pk"), tempfile!("a.sk"));
+    let peer_a = config::Rosenpass {
+        config_file_path: tempfile!("a.config"),
+        keypair: Some(peer_a_keypair.clone()),
+        listen: peer_a_endpoint.to_socket_addrs()?.collect(), // TODO: This could collide by accident
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("a.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("b.pk"),
+            key_out: Some(peer_a_osk.clone()),
+            endpoint: None,
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    let peer_b_keypair = config::Keypair::new(tempfile!("b.pk"), tempfile!("b.sk"));
+    let peer_b = config::Rosenpass {
+        config_file_path: tempfile!("b.config"),
+        keypair: Some(peer_b_keypair.clone()),
+        listen: vec![],
+        verbosity: config::Verbosity::Verbose,
+        api: api::config::ApiConfig {
+            listen_path: vec![tempfile!("b.sock")],
+            listen_fd: vec![],
+            stream_fd: vec![],
+        },
+        peers: vec![config::RosenpassPeer {
+            public_key: tempfile!("a.pk"),
+            key_out: Some(peer_b_osk.clone()),
+            endpoint: Some(peer_a_endpoint.to_owned()),
+            pre_shared_key: None,
+            wg: None,
+        }],
+    };
+
+    // Generate the keys
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_a_keypair.secret_key.clone(),
+        peer_a_keypair.public_key.clone(),
+    )?;
+    rosenpass::cli::testing::generate_and_save_keypair(
+        peer_b_keypair.secret_key.clone(),
+        peer_b_keypair.public_key.clone(),
+    )?;
+
+    // Write the configuration files
+    peer_a.commit()?;
+    peer_b.commit()?;
+
+    // Start peer a
+    let mut proc_a = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_a.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Start peer b
+    let mut proc_b = KillChild(
+        std::process::Command::new(env!("CARGO_BIN_EXE_rosenpass"))
+            .args([
+                "exchange-config",
+                peer_b.config_file_path.to_str().context("")?,
+            ])
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .spawn()?,
+    );
+
+    // Acquire stdout
+    let mut out_a = BufReader::new(proc_a.0.stdout.take().context("")?).lines();
+    let mut out_b = BufReader::new(proc_b.0.stdout.take().context("")?).lines();
+
+    // Wait for the keys to successfully exchange a key
+    let mut attempt = 0;
+    loop {
+        let line_a = out_a.next().context("")??;
+        let line_b = out_b.next().context("")??;
+
+        let words_a = line_a.split(' ').collect::<Vec<_>>();
+        let words_b = line_b.split(' ').collect::<Vec<_>>();
+
+        // FIXED     FIXED PEER-ID                                      FIXED    FILENAME       STATUS
+        // output-key peer KZqXTZ4l2aNnkJtLPhs4D8JxHTGmRSL9w3Qr+X8JxFk= key-file "client-A-osk" exchanged
+        let peer_a_id = words_b
+            .get(2)
+            .with_context(|| format!("Bad rosenpass output: `{line_b}`"))?;
+        let peer_b_id = words_a
+            .get(2)
+            .with_context(|| format!("Bad rosenpass output: `{line_a}`"))?;
+        assert_eq!(
+            line_a,
+            format!(
+                "output-key peer {peer_b_id} key-file \"{}\" exchanged",
+                peer_a_osk.to_str().context("")?
+            )
+        );
+        assert_eq!(
+            line_b,
+            format!(
+                "output-key peer {peer_a_id} key-file \"{}\" exchanged",
+                peer_b_osk.to_str().context("")?
+            )
+        );
+
+        // Read OSKs
+        let osk_a = SymKey::load_b64::<64, _>(peer_a_osk.clone())?;
+        let osk_b = SymKey::load_b64::<64, _>(peer_b_osk.clone())?;
+        match osk_a.secret() == osk_b.secret() {
+            true => break,
+            false if attempt > 10 => bail!("Peers did not produce a matching key even after ten attempts. Something is wrong with the key exchange!"),
+            false => {},
+        };
+
+        attempt += 1;
+    }
+
+    // Now connect to the peers
+    let api_a = UnixStream::connect(&peer_a.api.listen_path[0])?;
+    let api_b = UnixStream::connect(&peer_b.api.listen_path[0])?;
+
+    for conn in ([api_a, api_b]).iter() {
+        let mut echo = [0u8; 256];
+        copy_slice_least_src("Hello World".as_bytes()).to(&mut echo);
+
+        let req = api::PingRequest::new(echo);
+        LengthPrefixEncoder::from_message(req.as_bytes()).write_all_to_stdio(conn)?;
+
+        let mut decoder = LengthPrefixDecoder::new([0u8; api::MAX_RESPONSE_LEN]);
+        let res = decoder.read_all_from_stdio(conn)?;
+        let res = res.zk_parse::<api::PingResponse>()?;
+        assert_eq!(*res, api::PingResponse::new(echo));
+    }
+
+    Ok(())
+}

rosenpass/tests/integration_test.rs

@@ -6,7 +6,7 @@ use std::{
     time::Duration,
 };
 
-use clap::{builder::Str, Parser};
+use clap::Parser;
 use rosenpass::{app_server::AppServerTestBuilder, cli::CliArgs};
 use rosenpass_secret_memory::{Public, Secret};
 use rosenpass_wireguard_broker::{WireguardBrokerMio, WG_KEY_LEN, WG_PEER_LEN};
@@ -15,9 +15,19 @@ use std::io::Write;
 
 const BIN: &str = "rosenpass";
 
+fn setup_tests() {
+    use rosenpass_secret_memory as SM;
+    #[cfg(feature = "experiment_memfd_secret")]
+    SM::secret_policy_try_use_memfd_secrets();
+    #[cfg(not(feature = "experiment_memfd_secret"))]
+    SM::secret_policy_use_only_malloc_secrets();
+}
+
 // check that we can generate keys
 #[test]
 fn generate_keys() {
+    setup_tests();
+
     let tmpdir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("keygen");
     fs::create_dir_all(&tmpdir).unwrap();
 
@@ -94,14 +104,11 @@ fn run_server_client_exchange(
     .unwrap();
 
     std::thread::spawn(move || {
-        cli.command
-            .run(Some(
-                server_test_builder
+        let test_helpers = server_test_builder
             .termination_handler(Some(server_terminate_rx))
             .build()
-                    .unwrap(),
-            ))
             .unwrap();
+        cli.run(None, Some(test_helpers)).unwrap();
     });
 
     let cli = CliArgs::try_parse_from(
@@ -112,14 +119,11 @@ fn run_server_client_exchange(
     .unwrap();
 
     std::thread::spawn(move || {
-        cli.command
-            .run(Some(
-                client_test_builder
+        let test_helpers = client_test_builder
             .termination_handler(Some(client_terminate_rx))
             .build()
-                    .unwrap(),
-            ))
             .unwrap();
+        cli.run(None, Some(test_helpers)).unwrap();
     });
 
     // give them some time to do the key exchange under load
@@ -134,6 +138,7 @@ fn run_server_client_exchange(
 #[test]
 #[serial]
 fn check_exchange_under_normal() {
+    setup_tests();
     setup_logging();
 
     let tmpdir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("exchange");
@@ -206,6 +211,7 @@ fn check_exchange_under_normal() {
 #[test]
 #[serial]
 fn check_exchange_under_dos() {
+    setup_tests();
     setup_logging();
 
     //Generate binary with responder with feature integration_test
@@ -275,6 +281,7 @@ fn check_exchange_under_dos() {
     fs::remove_dir_all(&tmpdir).unwrap();
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Default)]
 struct MockBrokerInner {
     psk: Option<Secret<WG_KEY_LEN>>,
@@ -282,9 +289,11 @@ struct MockBrokerInner {
     interface: Option<String>,
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Default)]
 struct MockBroker {
     inner: Arc<Mutex<MockBrokerInner>>,
+    mio_token: Option<mio::Token>,
 }
 
 impl WireguardBrokerMio for MockBroker {
@@ -293,8 +302,9 @@ impl WireguardBrokerMio for MockBroker {
     fn register(
         &mut self,
         _registry: &mio::Registry,
-        _token: mio::Token,
+        token: mio::Token,
     ) -> Result<(), Self::MioError> {
+        self.mio_token = Some(token);
         Ok(())
     }
 
@@ -303,8 +313,13 @@ impl WireguardBrokerMio for MockBroker {
     }
 
     fn unregister(&mut self, _registry: &mio::Registry) -> Result<(), Self::MioError> {
+        self.mio_token = None;
         Ok(())
     }
+
+    fn mio_token(&self) -> Option<mio::Token> {
+        self.mio_token
+    }
 }
 
 impl rosenpass_wireguard_broker::WireGuardBroker for MockBroker {
@@ -320,7 +335,7 @@ impl rosenpass_wireguard_broker::WireGuardBroker for MockBroker {
             if let Ok(ref mut mutex) = lock {
                 **mutex = MockBrokerInner {
                     psk: Some(config.psk.clone()),
-                    peer_id: Some(config.peer_id.clone()),
+                    peer_id: Some(*config.peer_id),
                     interface: Some(std::str::from_utf8(config.interface).unwrap().to_string()),
                 };
                 break Ok(());

rp/Cargo.toml

@@ -37,3 +37,7 @@ netlink-packet-wireguard = "0.2"
 [dev-dependencies]
 tempfile = {workspace = true}
 stacker = {workspace = true}
+
+[features]
+experiment_memfd_secret = []
+experiment_libcrux = ["rosenpass-ciphers/experiment_libcrux"]

rp/src/exchange.rs

@@ -2,7 +2,9 @@ use std::{net::SocketAddr, path::PathBuf};
 
 use anyhow::Result;
 
+#[cfg(any(target_os = "linux", target_os = "freebsd"))]
 use crate::key::WG_B64_LEN;
+
 #[derive(Default)]
 pub struct ExchangePeer {
     pub public_keys_dir: PathBuf,
@@ -186,8 +188,7 @@ pub async fn exchange(options: ExchangeOptions) -> Result<()> {
     let pk = SPk::load(&pqpk)?;
 
     let mut srv = Box::new(AppServer::new(
-        sk,
-        pk,
+        Some((sk, pk)),
         if let Some(listen) = options.listen {
             vec![listen]
         } else {

rp/src/key.rs

@@ -1,11 +1,12 @@
 use std::{
     fs::{self, DirBuilder},
+    ops::DerefMut,
     os::unix::fs::{DirBuilderExt, PermissionsExt},
     path::Path,
 };
 
 use anyhow::{anyhow, Result};
-use rosenpass_util::file::{LoadValueB64, StoreValueB64};
+use rosenpass_util::file::{LoadValueB64, StoreValue, StoreValueB64};
 use zeroize::Zeroize;
 
 use rosenpass::protocol::{SPk, SSk};
@@ -56,8 +57,8 @@ pub fn genkey(private_keys_dir: &Path) -> Result<()> {
     if !pqsk_path.exists() && !pqpk_path.exists() {
         let mut pqsk = SSk::random();
         let mut pqpk = SPk::random();
-        StaticKem::keygen(pqsk.secret_mut(), pqpk.secret_mut())?;
-        pqpk.store_secret(pqpk_path)?;
+        StaticKem::keygen(pqsk.secret_mut(), pqpk.deref_mut())?;
+        pqpk.store(pqpk_path)?;
         pqsk.store_secret(pqsk_path)?;
     } else {
         eprintln!(
@@ -102,6 +103,7 @@ mod tests {
     use std::fs;
 
     use rosenpass::protocol::{SPk, SSk};
+    use rosenpass_secret_memory::secret_policy_try_use_memfd_secrets;
     use rosenpass_secret_memory::Secret;
     use rosenpass_util::file::LoadValue;
     use rosenpass_util::file::LoadValueB64;
@@ -110,7 +112,8 @@ mod tests {
     use crate::key::{genkey, pubkey, WG_B64_LEN};
 
     #[test]
-    fn it_works() {
+    fn test_key_loopback() {
+        secret_policy_try_use_memfd_secrets();
         let private_keys_dir = tempdir().unwrap();
         fs::remove_dir(private_keys_dir.path()).unwrap();
 

rp/src/main.rs

@@ -3,6 +3,7 @@ use std::process::exit;
 use cli::{Cli, Command};
 use exchange::exchange;
 use key::{genkey, pubkey};
+use rosenpass_secret_memory::policy;
 
 mod cli;
 mod exchange;
@@ -10,6 +11,11 @@ mod key;
 
 #[tokio::main]
 async fn main() {
+    #[cfg(feature = "experiment_memfd_secret")]
+    policy::secret_policy_try_use_memfd_secrets();
+    #[cfg(not(feature = "experiment_memfd_secret"))]
+    policy::secret_policy_use_only_malloc_secrets();
+
     let cli = match Cli::parse(std::env::args().peekable()) {
         Ok(cli) => cli,
         Err(err) => {

secret-memory/Cargo.toml

@@ -23,3 +23,4 @@ log = { workspace = true }
 allocator-api2-tests = { workspace = true }
 tempfile = {workspace = true}
 base64ct = {workspace = true}
+procspawn = {workspace = true}

secret-memory/src/alloc/memsec/malloc.rs

@@ -4,37 +4,41 @@ use std::ptr::NonNull;
 use allocator_api2::alloc::{AllocError, Allocator, Layout};
 
 #[derive(Copy, Clone, Default)]
-struct MemsecAllocatorContents;
+struct MallocAllocatorContents;
 
 /// Memory allocation using using the memsec crate
 #[derive(Copy, Clone, Default)]
-pub struct MemsecAllocator {
-    _dummy_private_data: MemsecAllocatorContents,
+pub struct MallocAllocator {
+    _dummy_private_data: MallocAllocatorContents,
 }
 
 /// A box backed by the memsec allocator
-pub type MemsecBox<T> = allocator_api2::boxed::Box<T, MemsecAllocator>;
+pub type MallocBox<T> = allocator_api2::boxed::Box<T, MallocAllocator>;
 
 /// A vector backed by the memsec allocator
-pub type MemsecVec<T> = allocator_api2::vec::Vec<T, MemsecAllocator>;
+pub type MallocVec<T> = allocator_api2::vec::Vec<T, MallocAllocator>;
 
-pub fn memsec_box<T>(x: T) -> MemsecBox<T> {
-    MemsecBox::<T>::new_in(x, MemsecAllocator::new())
+pub fn malloc_box_try<T>(x: T) -> Result<MallocBox<T>, AllocError> {
+    MallocBox::<T>::try_new_in(x, MallocAllocator::new())
 }
 
-pub fn memsec_vec<T>() -> MemsecVec<T> {
-    MemsecVec::<T>::new_in(MemsecAllocator::new())
+pub fn malloc_box<T>(x: T) -> MallocBox<T> {
+    MallocBox::<T>::new_in(x, MallocAllocator::new())
 }
 
-impl MemsecAllocator {
+pub fn malloc_vec<T>() -> MallocVec<T> {
+    MallocVec::<T>::new_in(MallocAllocator::new())
+}
+
+impl MallocAllocator {
     pub fn new() -> Self {
         Self {
-            _dummy_private_data: MemsecAllocatorContents,
+            _dummy_private_data: MallocAllocatorContents,
         }
     }
 }
 
-unsafe impl Allocator for MemsecAllocator {
+unsafe impl Allocator for MallocAllocator {
     fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
         // Call memsec allocator
         let mem: Option<NonNull<[u8]>> = unsafe { memsec::malloc_sized(layout.size()) };
@@ -48,8 +52,8 @@ unsafe impl Allocator for MemsecAllocator {
         // Ensure the right alignment is used
         let off = (mem.as_ptr() as *const u8).align_offset(layout.align());
         if off != 0 {
-            log::error!("Allocation {layout:?} was requested but memsec returned allocation \
-                with offset {off} from the requested alignment. Memsec always allocates values \
+            log::error!("Allocation {layout:?} was requested but malloc-based memsec returned allocation \
+                with offset {off} from the requested alignment. Malloc always allocates values \
                 at the end of a memory page for security reasons, custom alignments are not supported. \
                 You could try allocating an oversized value.");
             unsafe { memsec::free(mem) };
@@ -66,7 +70,7 @@ unsafe impl Allocator for MemsecAllocator {
     }
 }
 
-impl fmt::Debug for MemsecAllocator {
+impl fmt::Debug for MallocAllocator {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         fmt.write_str("<memsec based Rust allocator>")
     }
@@ -78,21 +82,21 @@ mod test {
 
     use super::*;
 
-    make_test! { test_sizes(MemsecAllocator::new()) }
-    make_test! { test_vec(MemsecAllocator::new()) }
-    make_test! { test_many_boxes(MemsecAllocator::new()) }
+    make_test! { test_sizes(MallocAllocator::new()) }
+    make_test! { test_vec(MallocAllocator::new()) }
+    make_test! { test_many_boxes(MallocAllocator::new()) }
 
     #[test]
-    fn memsec_allocation() {
-        let alloc = MemsecAllocator::new();
-        memsec_allocation_impl::<0>(&alloc);
-        memsec_allocation_impl::<7>(&alloc);
-        memsec_allocation_impl::<8>(&alloc);
-        memsec_allocation_impl::<64>(&alloc);
-        memsec_allocation_impl::<999>(&alloc);
+    fn malloc_allocation() {
+        let alloc = MallocAllocator::new();
+        malloc_allocation_impl::<0>(&alloc);
+        malloc_allocation_impl::<7>(&alloc);
+        malloc_allocation_impl::<8>(&alloc);
+        malloc_allocation_impl::<64>(&alloc);
+        malloc_allocation_impl::<999>(&alloc);
     }
 
-    fn memsec_allocation_impl<const N: usize>(alloc: &MemsecAllocator) {
+    fn malloc_allocation_impl<const N: usize>(alloc: &MallocAllocator) {
         let layout = Layout::new::<[u8; N]>();
         let mem = alloc.allocate(layout).unwrap();
 

secret-memory/src/alloc/memsec/memfdsec.rs

@@ -0,0 +1,112 @@
+#![cfg(target_os = "linux")]
+use std::fmt;
+use std::ptr::NonNull;
+
+use allocator_api2::alloc::{AllocError, Allocator, Layout};
+
+#[derive(Copy, Clone, Default)]
+struct MemfdSecAllocatorContents;
+
+/// Memory allocation using using the memsec crate
+#[derive(Copy, Clone, Default)]
+pub struct MemfdSecAllocator {
+    _dummy_private_data: MemfdSecAllocatorContents,
+}
+
+/// A box backed by the memsec allocator
+pub type MemfdSecBox<T> = allocator_api2::boxed::Box<T, MemfdSecAllocator>;
+
+/// A vector backed by the memsec allocator
+pub type MemfdSecVec<T> = allocator_api2::vec::Vec<T, MemfdSecAllocator>;
+
+pub fn memfdsec_box_try<T>(x: T) -> Result<MemfdSecBox<T>, AllocError> {
+    MemfdSecBox::<T>::try_new_in(x, MemfdSecAllocator::new())
+}
+
+pub fn memfdsec_box<T>(x: T) -> MemfdSecBox<T> {
+    MemfdSecBox::<T>::new_in(x, MemfdSecAllocator::new())
+}
+
+pub fn memfdsec_vec<T>() -> MemfdSecVec<T> {
+    MemfdSecVec::<T>::new_in(MemfdSecAllocator::new())
+}
+
+impl MemfdSecAllocator {
+    pub fn new() -> Self {
+        Self {
+            _dummy_private_data: MemfdSecAllocatorContents,
+        }
+    }
+}
+
+unsafe impl Allocator for MemfdSecAllocator {
+    fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+        // Call memsec allocator
+        let mem: Option<NonNull<[u8]>> = unsafe { memsec::memfd_secret_sized(layout.size()) };
+
+        // Unwrap the option
+        let Some(mem) = mem else {
+            log::error!("Allocation {layout:?} was requested but memfd-based memsec returned a null pointer");
+            return Err(AllocError);
+        };
+
+        // Ensure the right alignment is used
+        let off = (mem.as_ptr() as *const u8).align_offset(layout.align());
+        if off != 0 {
+            log::error!("Allocation {layout:?} was requested but memfd-based memsec returned allocation \
+                with offset {off} from the requested alignment. Memfd always allocates values \
+                at the end of a memory page for security reasons, custom alignments are not supported. \
+                You could try allocating an oversized value.");
+            unsafe { memsec::free_memfd_secret(mem) };
+            return Err(AllocError);
+        };
+
+        Ok(mem)
+    }
+
+    unsafe fn deallocate(&self, ptr: NonNull<u8>, _layout: Layout) {
+        unsafe {
+            memsec::free_memfd_secret(ptr);
+        }
+    }
+}
+
+impl fmt::Debug for MemfdSecAllocator {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt.write_str("<memsec based Rust allocator>")
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use allocator_api2_tests::make_test;
+
+    use super::*;
+
+    make_test! { test_sizes(MemfdSecAllocator::new()) }
+    make_test! { test_vec(MemfdSecAllocator::new()) }
+    make_test! { test_many_boxes(MemfdSecAllocator::new()) }
+
+    #[test]
+    fn memfdsec_allocation() {
+        let alloc = MemfdSecAllocator::new();
+        memfdsec_allocation_impl::<0>(&alloc);
+        memfdsec_allocation_impl::<7>(&alloc);
+        memfdsec_allocation_impl::<8>(&alloc);
+        memfdsec_allocation_impl::<64>(&alloc);
+        memfdsec_allocation_impl::<999>(&alloc);
+    }
+
+    fn memfdsec_allocation_impl<const N: usize>(alloc: &MemfdSecAllocator) {
+        let layout = Layout::new::<[u8; N]>();
+        let mem = alloc.allocate(layout).unwrap();
+
+        // https://libsodium.gitbook.io/doc/memory_management#guarded-heap-allocations
+        // promises us that allocated memory is initialized with the magic byte 0xDB
+        // and memsec promises to provide a reimplementation of the libsodium mechanism;
+        // it uses the magic value 0xD0 though
+        assert_eq!(unsafe { mem.as_ref() }, &[0xD0u8; N]);
+        let mem = NonNull::new(mem.as_ptr() as *mut u8).unwrap();
+        unsafe { alloc.deallocate(mem, layout) };
+    }
+}

secret-memory/src/alloc/memsec/mod.rs

@@ -0,0 +1,2 @@
+pub mod malloc;
+pub mod memfdsec;

secret-memory/src/alloc/mod.rs

@@ -1,6 +1,86 @@
 pub mod memsec;
 
-pub use crate::alloc::memsec::{
-    memsec_box as secret_box, memsec_vec as secret_vec, MemsecAllocator as SecretAllocator,
-    MemsecBox as SecretBox, MemsecVec as SecretVec,
-};
+use std::sync::OnceLock;
+
+use allocator_api2::alloc::{AllocError, Allocator};
+use memsec::malloc::MallocAllocator;
+
+#[cfg(target_os = "linux")]
+use memsec::memfdsec::MemfdSecAllocator;
+
+static ALLOC_TYPE: OnceLock<SecretAllocType> = OnceLock::new();
+
+/// Sets the secret allocation type to use.
+/// Intended usage at startup before secret allocation
+/// takes place
+pub fn set_secret_alloc_type(alloc_type: SecretAllocType) {
+    ALLOC_TYPE.set(alloc_type).unwrap();
+}
+
+pub fn get_or_init_secret_alloc_type(alloc_type: SecretAllocType) -> SecretAllocType {
+    *ALLOC_TYPE.get_or_init(|| alloc_type)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SecretAllocType {
+    MemsecMalloc,
+    #[cfg(target_os = "linux")]
+    MemsecMemfdSec,
+}
+
+pub struct SecretAlloc {
+    alloc_type: SecretAllocType,
+}
+
+impl Default for SecretAlloc {
+    fn default() -> Self {
+        Self {
+            alloc_type: *ALLOC_TYPE.get().expect(
+                "Secret security policy not specified. \
+                Run the specifying policy function in \
+                 rosenpass_secret_memory::policy or set a \
+                 custom policy by initializing \
+                 rosenpass_secret_memory::alloc::ALLOC_TYPE \
+                 before using secrets",
+            ),
+        }
+    }
+}
+
+unsafe impl Allocator for SecretAlloc {
+    fn allocate(
+        &self,
+        layout: std::alloc::Layout,
+    ) -> Result<std::ptr::NonNull<[u8]>, allocator_api2::alloc::AllocError> {
+        match self.alloc_type {
+            SecretAllocType::MemsecMalloc => MallocAllocator::default().allocate(layout),
+            #[cfg(target_os = "linux")]
+            SecretAllocType::MemsecMemfdSec => MemfdSecAllocator::default().allocate(layout),
+        }
+    }
+
+    unsafe fn deallocate(&self, ptr: std::ptr::NonNull<u8>, layout: std::alloc::Layout) {
+        match self.alloc_type {
+            SecretAllocType::MemsecMalloc => MallocAllocator::default().deallocate(ptr, layout),
+            #[cfg(target_os = "linux")]
+            SecretAllocType::MemsecMemfdSec => MemfdSecAllocator::default().deallocate(ptr, layout),
+        }
+    }
+}
+
+pub type SecretBox<T> = allocator_api2::boxed::Box<T, SecretAlloc>;
+
+/// A vector backed by the memsec allocator
+pub type SecretVec<T> = allocator_api2::vec::Vec<T, SecretAlloc>;
+
+pub fn secret_box_try<T>(x: T) -> Result<SecretBox<T>, AllocError> {
+    SecretBox::<T>::try_new_in(x, SecretAlloc::default())
+}
+
+pub fn secret_box<T>(x: T) -> SecretBox<T> {
+    SecretBox::<T>::new_in(x, SecretAlloc::default())
+}
+
+pub fn secret_vec<T>() -> SecretVec<T> {
+    SecretVec::<T>::new_in(SecretAlloc::default())
+}

secret-memory/src/lib.rs

@@ -6,6 +6,10 @@ pub mod alloc;
 
 mod public;
 pub use crate::public::Public;
+pub use crate::public::PublicBox;
 
 mod secret;
 pub use crate::secret::Secret;
+
+pub mod policy;
+pub use crate::policy::*;

secret-memory/src/policy.rs

@@ -0,0 +1,82 @@
+pub fn secret_policy_try_use_memfd_secrets() {
+    let alloc_type = {
+        #[cfg(target_os = "linux")]
+        {
+            if crate::alloc::memsec::memfdsec::memfdsec_box_try(0u8).is_ok() {
+                crate::alloc::SecretAllocType::MemsecMemfdSec
+            } else {
+                crate::alloc::SecretAllocType::MemsecMalloc
+            }
+        }
+
+        #[cfg(not(target_os = "linux"))]
+        {
+            crate::alloc::SecretAllocType::MemsecMalloc
+        }
+    };
+    assert_eq!(
+        alloc_type,
+        crate::alloc::get_or_init_secret_alloc_type(alloc_type)
+    );
+
+    log::info!("Secrets will be allocated using {:?}", alloc_type);
+}
+
+#[cfg(target_os = "linux")]
+pub fn secret_policy_use_only_memfd_secrets() {
+    let alloc_type = crate::alloc::SecretAllocType::MemsecMemfdSec;
+
+    assert_eq!(
+        alloc_type,
+        crate::alloc::get_or_init_secret_alloc_type(alloc_type)
+    );
+
+    log::info!("Secrets will be allocated using {:?}", alloc_type);
+}
+
+pub fn secret_policy_use_only_malloc_secrets() {
+    let alloc_type = crate::alloc::SecretAllocType::MemsecMalloc;
+    assert_eq!(
+        alloc_type,
+        crate::alloc::get_or_init_secret_alloc_type(alloc_type)
+    );
+
+    log::info!("Secrets will be allocated using {:?}", alloc_type);
+}
+
+pub mod test {
+    #[macro_export]
+    macro_rules! test_spawn_process_with_policies {
+        ($body:block, $($f: expr),*) => {
+            $(
+                let handle = procspawn::spawn((), |_| {
+
+                $f();
+
+                $body
+
+                });
+                handle.join().unwrap();
+            )*
+            };
+        }
+
+    #[macro_export]
+    macro_rules! test_spawn_process_provided_policies {
+        ($body: block) => {
+            $crate::test_spawn_process_with_policies!(
+                $body,
+                $crate::policy::secret_policy_try_use_memfd_secrets,
+                $crate::secret_policy_use_only_malloc_secrets
+            );
+
+            #[cfg(target_os = "linux")]
+            {
+                $crate::test_spawn_process_with_policies!(
+                    $body,
+                    $crate::policy::secret_policy_use_only_memfd_secrets
+                );
+            }
+        };
+    }
+}

secret-memory/src/public.rs

@@ -172,12 +172,154 @@ impl<const N: usize> StoreValueB64Writer for Public<N> {
     }
 }
 
+#[derive(Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[repr(transparent)]
+pub struct PublicBox<const N: usize> {
+    pub inner: Box<Public<N>>,
+}
+
+impl<const N: usize> PublicBox<N> {
+    /// Create a new [PublicBox] from a byte slice
+    pub fn from_slice(value: &[u8]) -> Self {
+        Self {
+            inner: Box::new(Public::from_slice(value)),
+        }
+    }
+
+    /// Create a new [PublicBox] from a byte array
+    pub fn new(value: [u8; N]) -> Self {
+        Self {
+            inner: Box::new(Public::new(value)),
+        }
+    }
+
+    /// Create a zero initialized [PublicBox]
+    pub fn zero() -> Self {
+        Self {
+            inner: Box::new(Public::zero()),
+        }
+    }
+
+    /// Create a random initialized [PublicBox]
+    pub fn random() -> Self {
+        Self {
+            inner: Box::new(Public::random()),
+        }
+    }
+
+    /// Randomize all bytes in an existing [PublicBox]
+    pub fn randomize(&mut self) {
+        self.inner.randomize()
+    }
+}
+
+impl<const N: usize> Randomize for PublicBox<N> {
+    fn try_fill<R: Rng + ?Sized>(&mut self, rng: &mut R) -> Result<(), rand::Error> {
+        self.inner.try_fill(rng)
+    }
+}
+
+impl<const N: usize> fmt::Debug for PublicBox<N> {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        debug_crypto_array(&**self, fmt)
+    }
+}
+
+impl<const N: usize> Deref for PublicBox<N> {
+    type Target = [u8; N];
+
+    fn deref(&self) -> &[u8; N] {
+        self.inner.deref()
+    }
+}
+
+impl<const N: usize> DerefMut for PublicBox<N> {
+    fn deref_mut(&mut self) -> &mut [u8; N] {
+        self.inner.deref_mut()
+    }
+}
+
+impl<const N: usize> Borrow<[u8]> for PublicBox<N> {
+    fn borrow(&self) -> &[u8] {
+        self.deref()
+    }
+}
+
+impl<const N: usize> BorrowMut<[u8]> for PublicBox<N> {
+    fn borrow_mut(&mut self) -> &mut [u8] {
+        self.deref_mut()
+    }
+}
+
+impl<const N: usize> LoadValue for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    // This is implemented separately from Public to avoid allocating too much stack memory
+    fn load<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
+        let mut p = Self::random();
+        fopen_r(path)?.read_exact_to_end(p.deref_mut())?;
+        Ok(p)
+    }
+}
+
+impl<const N: usize> StoreValue for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store<P: AsRef<Path>>(&self, path: P) -> anyhow::Result<()> {
+        self.inner.store(path)
+    }
+}
+
+impl<const N: usize> LoadValueB64 for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    // This is implemented separately from Public to avoid allocating too much stack memory
+    fn load_b64<const F: usize, P: AsRef<Path>>(path: P) -> Result<Self, Self::Error>
+    where
+        Self: Sized,
+    {
+        // A vector is used here to ensure heap allocation without copy from stack
+        let mut f = vec![0u8; F];
+        let mut v = PublicBox::zero();
+        let p = path.as_ref();
+
+        let len = fopen_r(p)?
+            .read_slice_to_end(&mut f)
+            .with_context(|| format!("Could not load file {p:?}"))?;
+
+        b64_decode(&f[0..len], v.deref_mut())
+            .with_context(|| format!("Could not decode base64 file {p:?}"))?;
+
+        Ok(v)
+    }
+}
+
+impl<const N: usize> StoreValueB64 for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store_b64<const F: usize, P: AsRef<Path>>(&self, path: P) -> anyhow::Result<()> {
+        self.inner.store_b64::<F, P>(path)
+    }
+}
+
+impl<const N: usize> StoreValueB64Writer for PublicBox<N> {
+    type Error = anyhow::Error;
+
+    fn store_b64_writer<const F: usize, W: std::io::Write>(
+        &self,
+        writer: W,
+    ) -> Result<(), Self::Error> {
+        self.inner.store_b64_writer::<F, W>(writer)
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
     #[cfg(test)]
+    #[allow(clippy::module_inception)]
     mod tests {
-        use crate::Public;
+        use crate::{Public, PublicBox};
         use rosenpass_util::{
             b64::b64_encode,
             file::{
@@ -185,32 +327,35 @@ mod tests {
                 Visibility,
             },
         };
-        use std::{fs, os::unix::fs::PermissionsExt};
+        use std::{fs, ops::Deref, os::unix::fs::PermissionsExt};
         use tempfile::tempdir;
 
-        /// test loading a public from an example file, and then storing it again in a different file
-        #[test]
-        fn test_public_load_store() {
+        /// Number of bytes in payload for load and store tests
         const N: usize = 100;
 
+        /// Convenience function for running a load/store test
+        fn run_load_store_test<
+            T: LoadValue<Error = anyhow::Error>
+                + StoreValue<Error = anyhow::Error>
+                + Deref<Target = [u8; N]>,
+        >() {
             // Generate original random bytes
             let original_bytes: [u8; N] = [rand::random(); N];
 
             // Create a temporary directory
             let temp_dir = tempdir().unwrap();
 
-            // Store the original public to an example file in the temporary directory
+            // Store the original bytes to an example file in the temporary directory
             let example_file = temp_dir.path().join("example_file");
-            std::fs::write(example_file.clone(), &original_bytes).unwrap();
+            std::fs::write(&example_file, original_bytes).unwrap();
 
-            // Load the public from the example file
+            // Load the value from the example file into our generic type
+            let loaded_public = T::load(&example_file).unwrap();
 
-            let loaded_public = Public::load(&example_file).unwrap();
+            // Check that the loaded value matches the original bytes
+            assert_eq!(loaded_public.deref(), &original_bytes);
 
-            // Check that the loaded public matches the original bytes
-            assert_eq!(&loaded_public.value, &original_bytes);
-
-            // Store the loaded public to a different file in the temporary directory
+            // Store the loaded value to a different file in the temporary directory
             let new_file = temp_dir.path().join("new_file");
             loaded_public.store(&new_file).unwrap();
 
@@ -224,10 +369,13 @@ mod tests {
             assert_eq!(new_file_contents, original_file_contents);
         }
 
-        /// test loading a base64 encoded public from an example file, and then storing it again in a different file
-        #[test]
-        fn test_public_load_store_base64() {
-            const N: usize = 100;
+        /// Convenience function for running a base64 load/store test
+        fn run_base64_load_store_test<
+            T: LoadValueB64<Error = anyhow::Error>
+                + StoreValueB64<Error = anyhow::Error>
+                + StoreValueB64Writer<Error = anyhow::Error>
+                + Deref<Target = [u8; N]>,
+        >() {
             // Generate original random bytes
             let original_bytes: [u8; N] = [rand::random(); N];
             // Create a temporary directory
@@ -238,9 +386,9 @@ mod tests {
             std::fs::write(&example_file, encoded_public).unwrap();
 
             // Load the public from the example file
-            let loaded_public = Public::load_b64::<{ N * 2 }, _>(&example_file).unwrap();
+            let loaded_public = T::load_b64::<{ N * 2 }, _>(&example_file).unwrap();
             // Check that the loaded public matches the original bytes
-            assert_eq!(&loaded_public.value, &original_bytes);
+            assert_eq!(loaded_public.deref(), &original_bytes);
 
             // Store the loaded public to a different file in the temporary directory
             let new_file = temp_dir.path().join("new_file");
@@ -253,7 +401,7 @@ mod tests {
             // Check that the contents of the new file match the original file
             assert_eq!(new_file_contents, original_file_contents);
 
-            //Check new file permissions are public
+            // Check new file permissions are public
             let metadata = fs::metadata(&new_file).unwrap();
             assert_eq!(metadata.permissions().mode() & 0o000777, 0o644);
 
@@ -271,9 +419,35 @@ mod tests {
             // Check that the contents of the new file match the original file
             assert_eq!(new_file_contents, original_file_contents);
 
-            //Check new file permissions are public
+            // Check new file permissions are public
             let metadata = fs::metadata(&new_file).unwrap();
             assert_eq!(metadata.permissions().mode() & 0o000777, 0o644);
         }
+
+        /// Test loading a [Public] from an example file, and then storing it again in a new file
+        #[test]
+        fn test_public_load_store() {
+            run_load_store_test::<Public<N>>();
+        }
+
+        /// Test loading a [PublicBox] from an example file, and then storing it again in a new file
+        #[test]
+        fn test_public_box_load_store() {
+            run_load_store_test::<PublicBox<N>>();
+        }
+
+        /// Test loading a base64-encoded [Public] from an example file, and then storing it again
+        /// in a different file
+        #[test]
+        fn test_public_load_store_base64() {
+            run_base64_load_store_test::<Public<N>>();
+        }
+
+        /// Test loading a base64-encoded [PublicBox] from an example file, and then storing it
+        /// again in a different file
+        #[test]
+        fn test_public_box_load_store_base64() {
+            run_base64_load_store_test::<PublicBox<N>>();
+        }
     }
 }

secret-memory/src/secret.rs

@@ -1,6 +1,5 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
-use std::convert::TryInto;
 use std::fmt;
 use std::ops::{Deref, DerefMut};
 use std::path::Path;
@@ -321,32 +320,41 @@ impl<const N: usize> StoreSecret for Secret<N> {
 
 #[cfg(test)]
 mod test {
+    use crate::test_spawn_process_provided_policies;
+
     use super::*;
     use std::{fs, os::unix::fs::PermissionsExt};
     use tempfile::tempdir;
 
+    procspawn::enable_test_support!();
+
     /// check that we can alloc using the magic pool
     #[test]
     fn secret_memory_pool_take() {
+        test_spawn_process_provided_policies!({
             const N: usize = 0x100;
             let mut pool = SecretMemoryPool::new();
             let secret: ZeroizingSecretBox<[u8; N]> = pool.take();
             assert_eq!(secret.as_ref(), &[0; N]);
+        });
     }
 
     /// check that a secret lives, even if its [SecretMemoryPool] is deleted
     #[test]
     fn secret_memory_pool_drop() {
+        test_spawn_process_provided_policies!({
             const N: usize = 0x100;
             let mut pool = SecretMemoryPool::new();
             let secret: ZeroizingSecretBox<[u8; N]> = pool.take();
             std::mem::drop(pool);
             assert_eq!(secret.as_ref(), &[0; N]);
+        });
     }
 
     /// check that a secret can be reborn, freshly initialized with zero
     #[test]
     fn secret_memory_pool_release() {
+        test_spawn_process_provided_policies!({
             const N: usize = 1;
             let mut pool = SecretMemoryPool::new();
             let mut secret: ZeroizingSecretBox<[u8; N]> = pool.take();
@@ -361,11 +369,13 @@ mod test {
 
             // and that the secret was zeroized
             assert_eq!(new_secret.as_ref(), &[0; N]);
+        });
     }
 
     /// test loading a secret from an example file, and then storing it again in a different file
     #[test]
     fn test_secret_load_store() {
+        test_spawn_process_provided_policies!({
             const N: usize = 100;
 
             // Generate original random bytes
@@ -376,7 +386,7 @@ mod test {
 
             // Store the original secret to an example file in the temporary directory
             let example_file = temp_dir.path().join("example_file");
-        std::fs::write(example_file.clone(), &original_bytes).unwrap();
+            std::fs::write(&example_file, original_bytes).unwrap();
 
             // Load the secret from the example file
             let loaded_secret = Secret::load(&example_file).unwrap();
@@ -396,11 +406,13 @@ mod test {
 
             // Check that the contents of the new file match the original file
             assert_eq!(new_file_contents, original_file_contents);
+        });
     }
 
     /// test loading a base64 encoded secret from an example file, and then storing it again in a different file
     #[test]
     fn test_secret_load_store_base64() {
+        test_spawn_process_provided_policies!({
             const N: usize = 100;
             // Generate original random bytes
             let original_bytes: [u8; N] = [rand::random(); N];
@@ -449,5 +461,6 @@ mod test {
             //Check new file permissions are secret
             let metadata = fs::metadata(&new_file).unwrap();
             assert_eq!(metadata.permissions().mode() & 0o000777, 0o600);
+        });
     }
 }

util/Cargo.toml

@@ -16,5 +16,14 @@ base64ct = { workspace = true }
 anyhow = { workspace = true }
 typenum = { workspace = true }
 static_assertions = { workspace = true }
-rustix = {workspace = true}
-zeroize = {workspace = true}
+rustix = { workspace = true }
+zeroize = { workspace = true }
+zerocopy = { workspace = true }
+thiserror = { workspace = true }
+mio = { workspace = true }
+tempfile = { workspace = true }
+uds = { workspace = true, optional = true, features = ["mio_1xx"] }
+
+
+[features]
+experiment_file_descriptor_passing = ["uds"]

util/src/build.rs

@@ -0,0 +1,169 @@
+use crate::{
+    functional::ApplyExt,
+    mem::{SwapWithDefaultExt, SwapWithExt},
+};
+
+#[derive(thiserror::Error, Debug)]
+pub enum ConstructionSiteErectError<E> {
+    #[error("Construction site is void")]
+    IsVoid,
+    #[error("Construction is already built")]
+    AlreadyBuilt,
+    #[error("Other construction site error {0:?}")]
+    Other(#[from] E),
+}
+
+pub trait Build<T>: Sized {
+    type Error;
+    fn build(self) -> Result<T, Self::Error>;
+}
+
+#[derive(Debug)]
+pub enum ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    Void,
+    Builder(Builder),
+    Product(T),
+}
+
+impl<Builder, T> Default for ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    fn default() -> Self {
+        Self::Void
+    }
+}
+
+impl<Builder, T> ConstructionSite<Builder, T>
+where
+    Builder: Build<T>,
+{
+    pub fn void() -> Self {
+        Self::Void
+    }
+
+    pub fn new(builder: Builder) -> Self {
+        Self::Builder(builder)
+    }
+
+    pub fn from_product(value: T) -> Self {
+        Self::Product(value)
+    }
+
+    pub fn take(&mut self) -> Self {
+        self.swap_with_default()
+    }
+
+    pub fn modify_taken_with_return<R, F>(&mut self, f: F) -> R
+    where
+        F: FnOnce(Self) -> (Self, R),
+    {
+        let (site, res) = self.take().apply(f);
+        self.swap_with(site);
+        res
+    }
+
+    pub fn modify_taken<F>(&mut self, f: F)
+    where
+        F: FnOnce(Self) -> Self,
+    {
+        self.take().apply(f).swap_with_mut(self)
+    }
+
+    #[allow(clippy::result_unit_err)]
+    pub fn erect(&mut self) -> Result<(), ConstructionSiteErectError<Builder::Error>> {
+        self.modify_taken_with_return(|site| {
+            let builder = match site {
+                site @ Self::Void => return (site, Err(ConstructionSiteErectError::IsVoid)),
+                site @ Self::Product(_) => {
+                    return (site, Err(ConstructionSiteErectError::AlreadyBuilt))
+                }
+                Self::Builder(builder) => builder,
+            };
+
+            let product = match builder.build() {
+                Err(e) => {
+                    return (Self::void(), Err(ConstructionSiteErectError::Other(e)));
+                }
+                Ok(p) => p,
+            };
+
+            (Self::from_product(product), Ok(()))
+        })
+    }
+
+    /// Returns `true` if the construction site is [`Void`].
+    ///
+    /// [`Void`]: ConstructionSite::Void
+    #[must_use]
+    pub fn is_void(&self) -> bool {
+        matches!(self, Self::Void)
+    }
+
+    /// Returns `true` if the construction site is [`InProgress`].
+    ///
+    /// [`InProgress`]: ConstructionSite::InProgress
+    #[must_use]
+    pub fn in_progess(&self) -> bool {
+        matches!(self, Self::Builder(..))
+    }
+
+    /// Returns `true` if the construction site is [`Done`].
+    ///
+    /// [`Done`]: ConstructionSite::Done
+    #[must_use]
+    pub fn is_available(&self) -> bool {
+        matches!(self, Self::Product(..))
+    }
+
+    pub fn into_builder(self) -> Option<Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn builder_ref(&self) -> Option<&Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn builder_mut(&mut self) -> Option<&mut Builder> {
+        use ConstructionSite as S;
+        match self {
+            S::Builder(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn into_product(self) -> Option<T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn product_ref(&self) -> Option<&T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    pub fn product_mut(&mut self) -> Option<&mut T> {
+        use ConstructionSite as S;
+        match self {
+            S::Product(v) => Some(v),
+            _ => None,
+        }
+    }
+}

util/src/controlflow.rs

@@ -0,0 +1,149 @@
+/// A collection of control flow utility macros
+
+#[macro_export]
+/// A simple for loop to repeat a $body a number of times
+macro_rules! repeat {
+    ($times:expr, $body:expr) => {
+        for _ in 0..($times) {
+            $body
+        }
+    };
+}
+
+#[macro_export]
+/// Return unless the condition $cond is true, with return value $val, if given.
+macro_rules! return_unless {
+    ($cond:expr) => {
+        if !($cond) {
+            return;
+        }
+    };
+    ($cond:expr, $val:expr) => {
+        if !($cond) {
+            return $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Return if the condition $cond is true, with return value $val, if given.
+macro_rules! return_if {
+    ($cond:expr) => {
+        if $cond {
+            return;
+        }
+    };
+    ($cond:expr, $val:expr) => {
+        if $cond {
+            return $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Break unless the condition is true, from the loop with label $val, if given.
+macro_rules! break_if {
+    ($cond:expr) => {
+        if $cond {
+            break;
+        }
+    };
+    ($cond:expr, $val:tt) => {
+        if $cond {
+            break $val;
+        }
+    };
+}
+
+#[macro_export]
+/// Continue if the condition is true, in the loop with label $val, if given.
+macro_rules! continue_if {
+    ($cond:expr) => {
+        if $cond {
+            continue;
+        }
+    };
+    ($cond:expr, $val:tt) => {
+        if $cond {
+            continue $val;
+        }
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_repeat() {
+        let mut sum = 0;
+        repeat!(10, {
+            sum += 1;
+        });
+        assert_eq!(sum, 10);
+    }
+
+    #[test]
+    fn test_return_unless() {
+        fn test_fn() -> i32 {
+            return_unless!(true, 1);
+            0
+        }
+        assert_eq!(test_fn(), 0);
+
+        fn test_fn2() -> i32 {
+            return_unless!(false, 1);
+            0
+        }
+        assert_eq!(test_fn2(), 1);
+    }
+
+    #[test]
+    fn test_return_if() {
+        fn test_fn() -> i32 {
+            return_if!(true, 1);
+            0
+        }
+        assert_eq!(test_fn(), 1);
+
+        fn test_fn2() -> i32 {
+            return_if!(false, 1);
+            0
+        }
+        assert_eq!(test_fn2(), 0);
+    }
+
+    #[test]
+    fn test_break_if() {
+        let mut sum = 0;
+        for i in 0..10 {
+            break_if!(i == 5);
+            sum += 1;
+        }
+        assert_eq!(sum, 5);
+
+        let mut sum = 0;
+        'one: for _ in 0..10 {
+            for j in 0..20 {
+                break_if!(j == 5, 'one);
+                sum += 1;
+            }
+        }
+        assert_eq!(sum, 5);
+    }
+
+    #[test]
+    fn test_continue_if() {
+        let mut sum = 0;
+        for i in 0..10 {
+            continue_if!(i == 5);
+            sum += 1;
+        }
+        assert_eq!(sum, 9);
+
+        let mut sum = 0;
+        'one: for i in 0..10 {
+            continue_if!(i == 5, 'one);
+            sum += 1;
+        }
+        assert_eq!(sum, 9);
+    }
+}

util/src/fd.rs

@@ -1,12 +1,300 @@
-use std::os::fd::{OwnedFd, RawFd};
+use anyhow::bail;
+use rustix::{
+    fd::{AsFd, BorrowedFd, FromRawFd, OwnedFd, RawFd},
+    io::fcntl_dupfd_cloexec,
+};
 
-/// Clone some file descriptor
+use crate::{mem::Forgetting, result::OkExt};
+
+/// Prepare a file descriptor for use in Rust code.
 ///
-/// If the file descriptor is invalid, an error will be raised.
-pub fn claim_fd(fd: RawFd) -> anyhow::Result<OwnedFd> {
-    use rustix::{fd::BorrowedFd, io::dup};
 
-    // This is safe since [dup] will simply raise
-    let fd = unsafe { dup(BorrowedFd::borrow_raw(fd))? };
-    Ok(fd)
+/// Checks if the file descriptor is valid and duplicates it to a new file descriptor.
+/// The old file descriptor is masked to avoid potential use after free (on file descriptor)
+/// in case the given file descriptor is still used somewhere
+pub fn claim_fd(fd: RawFd) -> rustix::io::Result<OwnedFd> {
+    let new = clone_fd_cloexec(unsafe { BorrowedFd::borrow_raw(fd) })?;
+    mask_fd(fd)?;
+    Ok(new)
+}
+
+/// Prepare a file descriptor for use in Rust code.
+///
+/// Checks if the file descriptor is valid.
+///
+/// Unlike [claim_fd], this will reuse the same file descriptor identifier instead of masking it.
+pub fn claim_fd_inplace(fd: RawFd) -> rustix::io::Result<OwnedFd> {
+    let mut new = unsafe { OwnedFd::from_raw_fd(fd) };
+    let tmp = clone_fd_cloexec(&new)?;
+    clone_fd_to_cloexec(tmp, &mut new)?;
+    Ok(new)
+}
+
+pub fn mask_fd(fd: RawFd) -> rustix::io::Result<()> {
+    // Safety: because the OwnedFd resulting from OwnedFd::from_raw_fd is wrapped in a Forgetting,
+    // it never gets dropped, meaning that fd is never closed and thus outlives the OwnedFd
+    let mut owned = Forgetting::new(unsafe { OwnedFd::from_raw_fd(fd) });
+    clone_fd_to_cloexec(open_nullfd()?, &mut owned)
+}
+
+pub fn clone_fd_cloexec<Fd: AsFd>(fd: Fd) -> rustix::io::Result<OwnedFd> {
+    const MINFD: RawFd = 3; // Avoid stdin, stdout, and stderr
+    fcntl_dupfd_cloexec(fd, MINFD)
+}
+
+#[cfg(target_os = "linux")]
+pub fn clone_fd_to_cloexec<Fd: AsFd>(fd: Fd, new: &mut OwnedFd) -> rustix::io::Result<()> {
+    use rustix::io::{dup3, DupFlags};
+    dup3(fd, new, DupFlags::CLOEXEC)
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn clone_fd_to_cloexec<Fd: AsFd>(fd: Fd, new: &mut OwnedFd) -> rustix::io::Result<()> {
+    use rustix::io::{dup2, fcntl_setfd, FdFlags};
+    dup2(&fd, new)?;
+    fcntl_setfd(&new, FdFlags::CLOEXEC)
+}
+
+/// Open a "blocked" file descriptor. I.e. a file descriptor that is neither meant for reading nor
+/// writing
+pub fn open_nullfd() -> rustix::io::Result<OwnedFd> {
+    use rustix::fs::{open, Mode, OFlags};
+    // TODO: Add tests showing that this will throw errors on use
+    open("/dev/null", OFlags::CLOEXEC, Mode::empty())
+}
+
+/// Convert low level errors into std::io::Error
+pub trait IntoStdioErr {
+    type Target;
+    fn into_stdio_err(self) -> Self::Target;
+}
+
+impl IntoStdioErr for rustix::io::Errno {
+    type Target = std::io::Error;
+
+    fn into_stdio_err(self) -> Self::Target {
+        std::io::Error::from_raw_os_error(self.raw_os_error())
+    }
+}
+
+impl<T> IntoStdioErr for rustix::io::Result<T> {
+    type Target = std::io::Result<T>;
+
+    fn into_stdio_err(self) -> Self::Target {
+        self.map_err(IntoStdioErr::into_stdio_err)
+    }
+}
+
+/// Read and write directly from a file descriptor
+pub struct FdIo<Fd: AsFd>(pub Fd);
+
+impl<Fd: AsFd> std::io::Read for FdIo<Fd> {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        rustix::io::read(&self.0, buf).into_stdio_err()
+    }
+}
+
+impl<Fd: AsFd> std::io::Write for FdIo<Fd> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        rustix::io::write(&self.0, buf).into_stdio_err()
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+pub trait StatExt {
+    fn is_socket(&self) -> bool;
+}
+
+impl StatExt for rustix::fs::Stat {
+    fn is_socket(&self) -> bool {
+        use rustix::fs::FileType;
+        let ft = FileType::from_raw_mode(self.st_mode);
+        matches!(ft, FileType::Socket)
+    }
+}
+
+pub trait TryStatExt {
+    type Error;
+    fn is_socket(&self) -> Result<bool, Self::Error>;
+}
+
+impl<T> TryStatExt for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn is_socket(&self) -> Result<bool, Self::Error> {
+        rustix::fs::fstat(self)?.is_socket().ok()
+    }
+}
+
+pub trait GetSocketType {
+    type Error;
+    fn socket_type(&self) -> Result<rustix::net::SocketType, Self::Error>;
+    fn is_datagram_socket(&self) -> Result<bool, Self::Error> {
+        use rustix::net::SocketType;
+        matches!(self.socket_type()?, SocketType::DGRAM).ok()
+    }
+    fn is_stream_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.socket_type()? == rustix::net::SocketType::STREAM)
+    }
+}
+
+impl<T> GetSocketType for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn socket_type(&self) -> Result<rustix::net::SocketType, Self::Error> {
+        rustix::net::sockopt::get_socket_type(self)
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetSocketDomain {
+    type Error;
+    fn socket_domain(&self) -> Result<rustix::net::AddressFamily, Self::Error>;
+    fn socket_address_family(&self) -> Result<rustix::net::AddressFamily, Self::Error> {
+        self.socket_domain()
+    }
+    fn is_unix_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.socket_domain()? == rustix::net::AddressFamily::UNIX)
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetSocketDomain for T
+where
+    T: AsFd,
+{
+    type Error = rustix::io::Errno;
+
+    fn socket_domain(&self) -> Result<rustix::net::AddressFamily, Self::Error> {
+        rustix::net::sockopt::get_socket_domain(self)
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetUnixSocketType {
+    type Error;
+    fn is_unix_stream_socket(&self) -> Result<bool, Self::Error>;
+    fn demand_unix_stream_socket(&self) -> anyhow::Result<()>;
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetUnixSocketType for T
+where
+    T: GetSocketType + GetSocketDomain<Error = <T as GetSocketType>::Error>,
+    anyhow::Error: From<<T as GetSocketType>::Error>,
+{
+    type Error = <T as GetSocketType>::Error;
+
+    fn is_unix_stream_socket(&self) -> Result<bool, Self::Error> {
+        Ok(self.is_unix_socket()? && self.is_stream_socket()?)
+    }
+
+    fn demand_unix_stream_socket(&self) -> anyhow::Result<()> {
+        use rustix::net::AddressFamily as SA;
+        use rustix::net::SocketType as ST;
+        match (self.socket_domain()?, self.socket_type()?) {
+            (SA::UNIX, ST::STREAM) => Ok(()),
+            (SA::UNIX, mode) => bail!("Expected unix socket in stream mode, but mode is {mode:?}"),
+            (domain, _) => bail!("Expected unix socket, but socket domain is {domain:?} instead"),
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub trait GetSocketProtocol {
+    fn socket_protocol(&self) -> Result<Option<rustix::net::Protocol>, rustix::io::Errno>;
+    fn is_udp_socket(&self) -> Result<bool, rustix::io::Errno> {
+        self.socket_protocol()?
+            .map(|p| p == rustix::net::ipproto::UDP)
+            .unwrap_or(false)
+            .ok()
+    }
+    fn demand_udp_socket(&self) -> anyhow::Result<()> {
+        match self.socket_protocol() {
+            Ok(Some(rustix::net::ipproto::UDP)) => Ok(()),
+            Ok(Some(other_proto)) => {
+                bail!("Not a udp socket, instead socket protocol is: {other_proto:?}")
+            }
+            Ok(None) => bail!("getsockopt() returned empty value"),
+            Err(errno) => Err(errno.into_stdio_err())?,
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl<T> GetSocketProtocol for T
+where
+    T: AsFd,
+{
+    fn socket_protocol(&self) -> Result<Option<rustix::net::Protocol>, rustix::io::Errno> {
+        rustix::net::sockopt::get_socket_protocol(self)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::{read_to_string, File};
+    use std::io::{Read, Write};
+    use std::os::fd::IntoRawFd;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_claim_fd() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let file = File::create(path.clone()).unwrap();
+        let fd: RawFd = file.into_raw_fd();
+        let owned_fd = claim_fd(fd).unwrap();
+        let mut file = unsafe { File::from_raw_fd(owned_fd.into_raw_fd()) };
+        file.write_all(b"Hello, World!").unwrap();
+
+        let message = read_to_string(path).unwrap();
+        assert_eq!(message, "Hello, World!");
+    }
+
+    #[test]
+    #[should_panic(expected = "fd != u32::MAX as RawFd")]
+    fn test_claim_fd_invalid_neg() {
+        let fd: RawFd = -1;
+        let _ = claim_fd(fd);
+    }
+
+    #[test]
+    #[should_panic(expected = "fd != u32::MAX as RawFd")]
+    fn test_claim_fd_invalid_max() {
+        let fd: RawFd = i64::MAX as RawFd;
+        let _ = claim_fd(fd);
+    }
+
+    #[test]
+    fn test_open_nullfd_write() {
+        let nullfd = open_nullfd().unwrap();
+        let mut file = unsafe { File::from_raw_fd(nullfd.into_raw_fd()) };
+        let res = file.write_all(b"Hello, World!");
+        assert!(res.is_err());
+        assert_eq!(
+            res.unwrap_err().to_string(),
+            "Bad file descriptor (os error 9)"
+        );
+    }
+
+    #[test]
+    fn test_open_nullfd_read() {
+        let nullfd = open_nullfd().unwrap();
+        let mut file = unsafe { File::from_raw_fd(nullfd.into_raw_fd()) };
+        let mut buffer = [0; 10];
+        let res = file.read_exact(&mut buffer);
+        assert!(res.is_err());
+        assert_eq!(res.unwrap_err().to_string(), "failed to fill whole buffer");
+    }
 }

util/src/file.rs

@@ -2,7 +2,6 @@ use anyhow::ensure;
 use std::fs::File;
 use std::io::Read;
 use std::os::unix::fs::OpenOptionsExt;
-use std::result::Result;
 use std::{fs::OpenOptions, path::Path};
 
 pub enum Visibility {
@@ -115,3 +114,96 @@ pub trait DisplayValueB64 {
 
     fn display_b64<'o>(&self, output: &'o mut [u8]) -> Result<&'o str, Self::Error>;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Write;
+    use std::os::unix::fs::PermissionsExt;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_fopen_w_public() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = fopen_w(path, Visibility::Public).unwrap();
+        file.write_all(b"test").unwrap();
+        let metadata = file.metadata().unwrap();
+        let permissions = metadata.permissions();
+        assert_eq!(permissions.mode(), 0o100644);
+    }
+
+    #[test]
+    fn test_fopen_w_secret() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = fopen_w(path, Visibility::Secret).unwrap();
+        file.write_all(b"test").unwrap();
+        let metadata = file.metadata().unwrap();
+        let permissions = metadata.permissions();
+        assert_eq!(permissions.mode(), 0o100600);
+    }
+
+    #[test]
+    fn test_fopen_r() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut contents = String::new();
+        let mut file = fopen_r(path).unwrap();
+        file.read_to_string(&mut contents).unwrap();
+        assert_eq!(contents, "test");
+    }
+
+    #[test]
+    fn test_read_slice_to_end() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 4];
+        let mut file = fopen_r(path).unwrap();
+        file.read_slice_to_end(&mut buf).unwrap();
+        assert_eq!(buf, [116, 101, 115, 116]);
+    }
+
+    #[test]
+    fn test_read_exact_to_end() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 4];
+        let mut file = fopen_r(path).unwrap();
+        file.read_exact_to_end(&mut buf).unwrap();
+        assert_eq!(buf, [116, 101, 115, 116]);
+    }
+
+    #[test]
+    fn test_read_exact_to_end_to_long() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 3];
+        let mut file = fopen_r(path).unwrap();
+        let result = file.read_exact_to_end(&mut buf);
+        assert!(result.is_err());
+        assert_eq!(result.unwrap_err().to_string(), "File too long!");
+    }
+
+    #[test]
+    fn test_read_slice_to_end_to_long() {
+        let tmp_dir = tempdir().unwrap();
+        let path = tmp_dir.path().join("test");
+        let mut file = File::create(path.clone()).unwrap();
+        file.write_all(b"test").unwrap();
+        let mut buf = [0u8; 3];
+        let mut file = fopen_r(path).unwrap();
+        let result = file.read_slice_to_end(&mut buf);
+        assert!(result.is_err());
+        assert_eq!(result.unwrap_err().to_string(), "File too long!");
+    }
+}

util/src/functional.rs

@@ -6,6 +6,32 @@ where
     v
 }
 
+pub trait MutatingExt {
+    fn mutating<F>(self, f: F) -> Self
+    where
+        F: Fn(&mut Self);
+    fn mutating_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&mut Self);
+}
+
+impl<T> MutatingExt for T {
+    fn mutating<F>(self, f: F) -> Self
+    where
+        F: Fn(&mut Self),
+    {
+        mutating(self, f)
+    }
+
+    fn mutating_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&mut Self),
+    {
+        f(self);
+        self
+    }
+}
+
 pub fn sideeffect<T, F>(v: T, f: F) -> T
 where
     F: Fn(&T),
@@ -13,3 +39,59 @@ where
     f(&v);
     v
 }
+
+pub trait SideffectExt {
+    fn sideeffect<F>(self, f: F) -> Self
+    where
+        F: Fn(&Self);
+    fn sideeffect_ref<F>(&self, f: F) -> &Self
+    where
+        F: Fn(&Self);
+    fn sideeffect_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&Self);
+}
+
+impl<T> SideffectExt for T {
+    fn sideeffect<F>(self, f: F) -> Self
+    where
+        F: Fn(&Self),
+    {
+        sideeffect(self, f)
+    }
+
+    fn sideeffect_ref<F>(&self, f: F) -> &Self
+    where
+        F: Fn(&Self),
+    {
+        f(self);
+        self
+    }
+
+    fn sideeffect_mut<F>(&mut self, f: F) -> &mut Self
+    where
+        F: Fn(&Self),
+    {
+        f(self);
+        self
+    }
+}
+
+pub fn run<R, F: FnOnce() -> R>(f: F) -> R {
+    f()
+}
+
+pub trait ApplyExt: Sized {
+    fn apply<R, F>(self, f: F) -> R
+    where
+        F: FnOnce(Self) -> R;
+}
+
+impl<T: Sized> ApplyExt for T {
+    fn apply<R, F>(self, f: F) -> R
+    where
+        F: FnOnce(Self) -> R,
+    {
+        f(self)
+    }
+}

util/src/io.rs

@@ -0,0 +1,180 @@
+use std::{borrow::Borrow, io};
+
+use anyhow::ensure;
+
+pub trait IoErrorKind {
+    fn io_error_kind(&self) -> io::ErrorKind;
+}
+
+impl<T: Borrow<io::Error>> IoErrorKind for T {
+    fn io_error_kind(&self) -> io::ErrorKind {
+        self.borrow().kind()
+    }
+}
+
+pub trait TryIoErrorKind {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind>;
+}
+
+impl<T: IoErrorKind> TryIoErrorKind for T {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind> {
+        Some(self.io_error_kind())
+    }
+}
+
+pub trait IoResultKindHintExt<T>: Sized {
+    type Error;
+    fn io_err_kind_hint(self) -> Result<T, (Self::Error, io::ErrorKind)>;
+}
+
+impl<T, E: IoErrorKind> IoResultKindHintExt<T> for Result<T, E> {
+    type Error = E;
+    fn io_err_kind_hint(self) -> Result<T, (E, io::ErrorKind)> {
+        self.map_err(|e| {
+            let kind = e.borrow().io_error_kind();
+            (e, kind)
+        })
+    }
+}
+
+pub trait TryIoResultKindHintExt<T>: Sized {
+    type Error;
+    fn try_io_err_kind_hint(self) -> Result<T, (Self::Error, Option<io::ErrorKind>)>;
+}
+
+impl<T, E: TryIoErrorKind> TryIoResultKindHintExt<T> for Result<T, E> {
+    type Error = E;
+    fn try_io_err_kind_hint(self) -> Result<T, (E, Option<io::ErrorKind>)> {
+        self.map_err(|e| {
+            let opt_kind = e.borrow().try_io_error_kind();
+            (e, opt_kind)
+        })
+    }
+}
+
+pub trait SubstituteForIoErrorKindExt<T>: Sized {
+    type Error;
+    fn substitute_for_ioerr_kind_with<F: FnOnce() -> T>(
+        self,
+        kind: io::ErrorKind,
+        f: F,
+    ) -> Result<T, Self::Error>;
+    fn substitute_for_ioerr_kind(self, kind: io::ErrorKind, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(kind, || v)
+    }
+
+    fn substitute_for_ioerr_interrupted_with<F: FnOnce() -> T>(
+        self,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(io::ErrorKind::Interrupted, f)
+    }
+
+    fn substitute_for_ioerr_interrupted(self, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_interrupted_with(|| v)
+    }
+
+    fn substitute_for_ioerr_wouldblock_with<F: FnOnce() -> T>(
+        self,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_kind_with(io::ErrorKind::WouldBlock, f)
+    }
+
+    fn substitute_for_ioerr_wouldblock(self, v: T) -> Result<T, Self::Error> {
+        self.substitute_for_ioerr_wouldblock_with(|| v)
+    }
+}
+
+impl<T, E: TryIoErrorKind> SubstituteForIoErrorKindExt<T> for Result<T, E> {
+    type Error = E;
+
+    fn substitute_for_ioerr_kind_with<F: FnOnce() -> T>(
+        self,
+        kind: io::ErrorKind,
+        f: F,
+    ) -> Result<T, Self::Error> {
+        match self.try_io_err_kind_hint() {
+            Ok(v) => Ok(v),
+            Err((_, Some(k))) if k == kind => Ok(f()),
+            Err((e, _)) => Err(e),
+        }
+    }
+}
+
+/// Automatically handles `std::io::ErrorKind::Interrupted`.
+///
+/// - If there is no error (i.e. on `Ok(r)`), the function will return `Ok(Some(r))`
+/// - `Interrupted` is handled internally, by retrying the IO operation
+/// - Other errors are returned as is
+pub fn handle_interrupted<R, E, F>(mut iofn: F) -> Result<Option<R>, E>
+where
+    E: TryIoErrorKind,
+    F: FnMut() -> Result<R, E>,
+{
+    use io::ErrorKind as E;
+    loop {
+        match iofn().try_io_err_kind_hint() {
+            Ok(v) => return Ok(Some(v)),
+            Err((_, Some(E::Interrupted))) => continue, // try again
+            Err((e, _)) => return Err(e),
+        };
+    }
+}
+
+/// Automatically handles `std::io::ErrorKind::{WouldBlock, Interrupted}`.
+///
+/// - If there is no error (i.e. on `Ok(r)`), the function will return `Ok(Some(r))`
+/// - `Interrupted` is handled internally, by retrying the IO operation
+/// - `WouldBlock` is handled by returning `Ok(None)`,
+/// - Other errors are returned as is
+pub fn nonblocking_handle_io_errors<R, E, F>(mut iofn: F) -> Result<Option<R>, E>
+where
+    E: TryIoErrorKind,
+    F: FnMut() -> Result<R, E>,
+{
+    use io::ErrorKind as E;
+    loop {
+        match iofn().try_io_err_kind_hint() {
+            Ok(v) => return Ok(Some(v)),
+            Err((_, Some(E::WouldBlock))) => return Ok(None), // no data to read
+            Err((_, Some(E::Interrupted))) => continue,       // try again
+            Err((e, _)) => return Err(e),
+        };
+    }
+}
+
+pub trait ReadNonblockingWithBoringErrorsHandledExt {
+    /// Convenience wrapper using [nonblocking_handle_io_errors] with [std::io::Read]
+    fn read_nonblocking_with_boring_errors_handled(
+        &mut self,
+        buf: &mut [u8],
+    ) -> io::Result<Option<usize>>;
+}
+
+impl<T: io::Read> ReadNonblockingWithBoringErrorsHandledExt for T {
+    fn read_nonblocking_with_boring_errors_handled(
+        &mut self,
+        buf: &mut [u8],
+    ) -> io::Result<Option<usize>> {
+        nonblocking_handle_io_errors(|| self.read(buf))
+    }
+}
+
+pub trait ReadExt {
+    fn read_exact_til_end(&mut self, buf: &mut [u8]) -> anyhow::Result<()>;
+}
+
+impl<T> ReadExt for T
+where
+    T: std::io::Read,
+{
+    fn read_exact_til_end(&mut self, buf: &mut [u8]) -> anyhow::Result<()> {
+        self.read_exact(buf)?;
+        ensure!(
+            self.read(&mut [0u8; 8])? == 0,
+            "Read source longer than buffer"
+        );
+        Ok(())
+    }
+}

util/src/length_prefix_encoding/decoder.rs

@@ -0,0 +1,359 @@
+use std::{borrow::BorrowMut, cmp::min, io};
+
+use thiserror::Error;
+use zeroize::Zeroize;
+
+use crate::{
+    io::{TryIoErrorKind, TryIoResultKindHintExt},
+    result::ensure_or,
+};
+
+pub const HEADER_SIZE: usize = std::mem::size_of::<u64>();
+
+#[derive(Error, Debug)]
+pub enum SanityError {
+    #[error("Offset is out of read buffer bounds")]
+    OutOfBufferBounds,
+    #[error("Offset is out of message buffer bounds")]
+    OutOfMessageBounds,
+}
+
+#[derive(Error, Debug)]
+#[error("Message too large ({msg_size} bytes) for buffer ({buf_size} bytes)")]
+pub struct MessageTooLargeError {
+    msg_size: usize,
+    buf_size: usize,
+}
+
+impl MessageTooLargeError {
+    pub fn new(msg_size: usize, buf_size: usize) -> Self {
+        Self { msg_size, buf_size }
+    }
+
+    pub fn ensure(msg_size: usize, buf_size: usize) -> Result<(), Self> {
+        let err = MessageTooLargeError { msg_size, buf_size };
+        ensure_or(msg_size <= buf_size, err)
+    }
+}
+
+#[derive(Debug)]
+pub struct ReadFromIoReturn<'a> {
+    pub bytes_read: usize,
+    pub message: Option<&'a mut [u8]>,
+}
+
+impl<'a> ReadFromIoReturn<'a> {
+    pub fn new(bytes_read: usize, message: Option<&'a mut [u8]>) -> Self {
+        Self {
+            bytes_read,
+            message,
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum ReadFromIoError {
+    #[error("Error reading from the underlying stream")]
+    IoError(#[from] io::Error),
+    #[error("Message size out of buffer bounds")]
+    MessageTooLargeError(#[from] MessageTooLargeError),
+}
+
+impl TryIoErrorKind for ReadFromIoError {
+    fn try_io_error_kind(&self) -> Option<io::ErrorKind> {
+        match self {
+            ReadFromIoError::IoError(ioe) => Some(ioe.kind()),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct LengthPrefixDecoder<Buf: BorrowMut<[u8]>> {
+    header: [u8; HEADER_SIZE],
+    buf: Buf,
+    off: usize,
+}
+
+impl<Buf: BorrowMut<[u8]>> LengthPrefixDecoder<Buf> {
+    pub fn new(buf: Buf) -> Self {
+        let header = Default::default();
+        let off = 0;
+        Self { header, buf, off }
+    }
+
+    pub fn clear(&mut self) {
+        self.zeroize()
+    }
+
+    pub fn from_parts(header: [u8; HEADER_SIZE], buf: Buf, off: usize) -> Self {
+        Self { header, buf, off }
+    }
+
+    pub fn into_parts(self) -> ([u8; HEADER_SIZE], Buf, usize) {
+        let Self { header, buf, off } = self;
+        (header, buf, off)
+    }
+
+    pub fn read_all_from_stdio<R: io::Read>(
+        &mut self,
+        mut r: R,
+    ) -> Result<&mut [u8], ReadFromIoError> {
+        use io::ErrorKind as K;
+        loop {
+            match self.read_from_stdio(&mut r).try_io_err_kind_hint() {
+                // Success (appeasing the borrow checker by calling message_mut())
+                Ok(ReadFromIoReturn {
+                    message: Some(_), ..
+                }) => break Ok(self.message_mut().unwrap().unwrap()),
+
+                // Unexpected EOF
+                Ok(ReadFromIoReturn { bytes_read: 0, .. }) => {
+                    break Err(ReadFromIoError::IoError(io::Error::new(
+                        K::UnexpectedEof,
+                        "",
+                    )))
+                }
+
+                // Retry
+                Ok(ReadFromIoReturn { message: None, .. }) => continue,
+                Err((_, Some(K::Interrupted))) => continue,
+
+                // Other error
+                Err((e, _)) => break Err(e),
+            }
+        }
+    }
+
+    pub fn read_from_stdio<R: io::Read>(
+        &mut self,
+        mut r: R,
+    ) -> Result<ReadFromIoReturn, ReadFromIoError> {
+        Ok(match self.next_slice_to_write_to()? {
+            // Read some bytes; any MessageTooLargeError in the call to self.message_mut() is
+            // ignored to ensure this function changes no state upon errors; the user should rerun
+            // the function and collect the MessageTooLargeError on the following invocation
+            Some(buf) => {
+                let bytes_read = r.read(buf)?;
+                self.advance(bytes_read).unwrap();
+                let message = self.message_mut().ok().flatten();
+                ReadFromIoReturn {
+                    bytes_read,
+                    message,
+                }
+            }
+            // Message is already fully read; full delegation to self.message_mut()
+            None => ReadFromIoReturn {
+                bytes_read: 0,
+                message: self.message_mut()?,
+            },
+        })
+    }
+
+    pub fn next_slice_to_write_to(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        fn some_if_nonempty(buf: &mut [u8]) -> Option<&mut [u8]> {
+            match buf.is_empty() {
+                true => None,
+                false => Some(buf),
+            }
+        }
+
+        macro_rules! return_if_nonempty_some {
+            ($opt:expr) => {{
+                // Deliberate double expansion of $opt to appease the borrow checker *sigh*
+                if $opt.and_then(some_if_nonempty).is_some() {
+                    return Ok($opt);
+                }
+            }};
+        }
+
+        return_if_nonempty_some!(Some(self.header_buffer_left_mut()));
+        return_if_nonempty_some!(self.message_fragment_left_mut()?);
+        Ok(None)
+    }
+
+    pub fn advance(&mut self, count: usize) -> Result<(), SanityError> {
+        let off = self.off + count;
+        let msg_off = off.saturating_sub(HEADER_SIZE);
+
+        use SanityError as E;
+        let alloc = self.message_buffer().len();
+        let msgsz = self.message_size();
+        ensure_or(msg_off <= alloc, E::OutOfBufferBounds)?;
+        ensure_or(
+            msgsz.map(|s| msg_off <= s).unwrap_or(true),
+            E::OutOfMessageBounds,
+        )?;
+
+        self.off = off;
+        Ok(())
+    }
+
+    pub fn ensure_sufficient_msg_buffer(&self) -> Result<(), MessageTooLargeError> {
+        let buf_size = self.message_buffer().len();
+        let msg_size = match self.get_header() {
+            None => return Ok(()),
+            Some(v) => v,
+        };
+        MessageTooLargeError::ensure(msg_size, buf_size)
+    }
+
+    pub fn header_buffer(&self) -> &[u8] {
+        &self.header[..]
+    }
+
+    pub fn header_buffer_mut(&mut self) -> &mut [u8] {
+        &mut self.header[..]
+    }
+
+    pub fn message_buffer(&self) -> &[u8] {
+        self.buf.borrow()
+    }
+
+    pub fn message_buffer_mut(&mut self) -> &mut [u8] {
+        self.buf.borrow_mut()
+    }
+
+    pub fn bytes_read(&self) -> &usize {
+        &self.off
+    }
+
+    pub fn into_message_buffer(self) -> Buf {
+        let Self { buf, .. } = self;
+        buf
+    }
+
+    pub fn header_buffer_offset(&self) -> usize {
+        min(self.off, HEADER_SIZE)
+    }
+
+    pub fn message_buffer_offset(&self) -> usize {
+        self.off.saturating_sub(HEADER_SIZE)
+    }
+
+    pub fn has_header(&self) -> bool {
+        self.header_buffer_offset() == HEADER_SIZE
+    }
+
+    pub fn has_message(&self) -> Result<bool, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        let msg_size = match self.get_header() {
+            None => return Ok(false),
+            Some(v) => v,
+        };
+        Ok(self.message_buffer_avail().len() == msg_size)
+    }
+
+    pub fn header_buffer_avail(&self) -> &[u8] {
+        let off = self.header_buffer_offset();
+        &self.header_buffer()[..off]
+    }
+
+    pub fn header_buffer_avail_mut(&mut self) -> &mut [u8] {
+        let off = self.header_buffer_offset();
+        &mut self.header_buffer_mut()[..off]
+    }
+
+    pub fn header_buffer_left(&self) -> &[u8] {
+        let off = self.header_buffer_offset();
+        &self.header_buffer()[off..]
+    }
+
+    pub fn header_buffer_left_mut(&mut self) -> &mut [u8] {
+        let off = self.header_buffer_offset();
+        &mut self.header_buffer_mut()[off..]
+    }
+
+    pub fn message_buffer_avail(&self) -> &[u8] {
+        let off = self.message_buffer_offset();
+        &self.message_buffer()[..off]
+    }
+
+    pub fn message_buffer_avail_mut(&mut self) -> &mut [u8] {
+        let off = self.message_buffer_offset();
+        &mut self.message_buffer_mut()[..off]
+    }
+
+    pub fn message_buffer_left(&self) -> &[u8] {
+        let off = self.message_buffer_offset();
+        &self.message_buffer()[off..]
+    }
+
+    pub fn message_buffer_left_mut(&mut self) -> &mut [u8] {
+        let off = self.message_buffer_offset();
+        &mut self.message_buffer_mut()[off..]
+    }
+
+    pub fn get_header(&self) -> Option<usize> {
+        match self.header_buffer_offset() == HEADER_SIZE {
+            false => None,
+            true => Some(u64::from_le_bytes(self.header) as usize),
+        }
+    }
+
+    pub fn message_size(&self) -> Option<usize> {
+        self.get_header()
+    }
+
+    pub fn encoded_message_bytes(&self) -> Option<usize> {
+        self.message_size().map(|sz| sz + HEADER_SIZE)
+    }
+
+    pub fn message_fragment(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        Ok(self.message_size().map(|sz| &self.message_buffer()[..sz]))
+    }
+
+    pub fn message_fragment_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        self.ensure_sufficient_msg_buffer()?;
+        Ok(self
+            .message_size()
+            .map(|sz| &mut self.message_buffer_mut()[..sz]))
+    }
+
+    pub fn message_fragment_avail(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment()
+            .map(|frag| frag.map(|frag| &frag[..off]))
+    }
+
+    pub fn message_fragment_avail_mut(
+        &mut self,
+    ) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment_mut()
+            .map(|frag| frag.map(|frag| &mut frag[..off]))
+    }
+
+    pub fn message_fragment_left(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment()
+            .map(|frag| frag.map(|frag| &frag[off..]))
+    }
+
+    pub fn message_fragment_left_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let off = self.message_buffer_avail().len();
+        self.message_fragment_mut()
+            .map(|frag| frag.map(|frag| &mut frag[off..]))
+    }
+
+    pub fn message(&self) -> Result<Option<&[u8]>, MessageTooLargeError> {
+        let sz = self.message_size();
+        self.message_fragment_avail()
+            .map(|frag_opt| frag_opt.and_then(|frag| (frag.len() == sz?).then_some(frag)))
+    }
+
+    pub fn message_mut(&mut self) -> Result<Option<&mut [u8]>, MessageTooLargeError> {
+        let sz = self.message_size();
+        self.message_fragment_avail_mut()
+            .map(|frag_opt| frag_opt.and_then(|frag| (frag.len() == sz?).then_some(frag)))
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> Zeroize for LengthPrefixDecoder<Buf> {
+    fn zeroize(&mut self) {
+        self.header.zeroize();
+        self.message_buffer_mut().zeroize();
+        self.off.zeroize();
+    }
+}

util/src/length_prefix_encoding/encoder.rs

@@ -0,0 +1,381 @@
+use std::{
+    borrow::{Borrow, BorrowMut},
+    cmp::min,
+    io,
+};
+
+use thiserror::Error;
+use zeroize::Zeroize;
+
+use crate::{io::IoResultKindHintExt, result::ensure_or};
+
+pub const HEADER_SIZE: usize = std::mem::size_of::<u64>();
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of buffer bounds")]
+pub struct PositionOutOfBufferBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of message bounds")]
+pub struct PositionOutOfMessageBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Write position is out of header bounds")]
+pub struct PositionOutOfHeaderBounds;
+
+#[derive(Error, Debug, Clone, Copy)]
+#[error("Message length is bigger than buffer length")]
+pub struct MessageTooLarge;
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum MessageLenSanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    MessageTooLarge(#[from] MessageTooLarge),
+}
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum PositionSanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    PositionOutOfBufferBounds(#[from] PositionOutOfBufferBounds),
+}
+
+#[derive(Error, Debug, Clone, Copy)]
+pub enum SanityError {
+    #[error("{0:?}")]
+    PositionOutOfMessageBounds(#[from] PositionOutOfMessageBounds),
+    #[error("{0:?}")]
+    PositionOutOfBufferBounds(#[from] PositionOutOfBufferBounds),
+    #[error("{0:?}")]
+    MessageTooLarge(#[from] MessageTooLarge),
+}
+
+impl TryFrom<SanityError> for MessageLenSanityError {
+    type Error = PositionOutOfBufferBounds;
+
+    fn try_from(value: SanityError) -> Result<Self, Self::Error> {
+        use {MessageLenSanityError as T, SanityError as F};
+        match value {
+            F::PositionOutOfMessageBounds(e) => Ok(T::PositionOutOfMessageBounds(e)),
+            F::MessageTooLarge(e) => Ok(T::MessageTooLarge(e)),
+            F::PositionOutOfBufferBounds(e) => Err(e),
+        }
+    }
+}
+
+impl From<MessageLenSanityError> for SanityError {
+    fn from(value: MessageLenSanityError) -> Self {
+        use {MessageLenSanityError as F, SanityError as T};
+        match value {
+            F::PositionOutOfMessageBounds(e) => T::PositionOutOfMessageBounds(e),
+            F::MessageTooLarge(e) => T::MessageTooLarge(e),
+        }
+    }
+}
+
+impl From<PositionSanityError> for SanityError {
+    fn from(value: PositionSanityError) -> Self {
+        use {PositionSanityError as F, SanityError as T};
+        match value {
+            F::PositionOutOfBufferBounds(e) => T::PositionOutOfBufferBounds(e),
+            F::PositionOutOfMessageBounds(e) => T::PositionOutOfMessageBounds(e),
+        }
+    }
+}
+
+pub struct WriteToIoReturn {
+    pub bytes_written: usize,
+    pub done: bool,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct LengthPrefixEncoder<Buf: Borrow<[u8]>> {
+    buf: Buf,
+    header: [u8; HEADER_SIZE],
+    pos: usize,
+}
+
+impl<Buf: Borrow<[u8]>> LengthPrefixEncoder<Buf> {
+    pub fn from_buffer(buf: Buf) -> Self {
+        let (header, pos) = ([0u8; HEADER_SIZE], 0);
+        let mut r = Self { buf, header, pos };
+        r.clear();
+        r
+    }
+
+    pub fn from_message(msg: Buf) -> Self {
+        let mut r = Self::from_buffer(msg);
+        r.restart_write_with_new_message(r.buffer_bytes().len())
+            .unwrap();
+        r
+    }
+
+    pub fn from_short_message(msg: Buf, len: usize) -> Result<Self, MessageLenSanityError> {
+        let mut r = Self::from_message(msg);
+        r.set_message_len(len)?;
+        Ok(r)
+    }
+
+    pub fn from_parts(buf: Buf, len: usize, pos: usize) -> Result<Self, SanityError> {
+        let mut r = Self::from_buffer(buf);
+        r.set_msg_len_and_position(len, pos)?;
+        Ok(r)
+    }
+
+    pub fn into_buffer(self) -> Buf {
+        let Self { buf, .. } = self;
+        buf
+    }
+
+    pub fn into_parts(self) -> (Buf, usize, usize) {
+        let len = self.message_len();
+        let pos = self.writing_position();
+        let buf = self.into_buffer();
+        (buf, len, pos)
+    }
+
+    pub fn clear(&mut self) {
+        self.set_msg_len_and_position(0, 0).unwrap();
+        self.set_message_offset(0).unwrap();
+    }
+
+    pub fn write_all_to_stdio<W: io::Write>(&mut self, mut w: W) -> io::Result<()> {
+        use io::ErrorKind as K;
+        loop {
+            match self.write_to_stdio(&mut w).io_err_kind_hint() {
+                // Done
+                Ok(WriteToIoReturn { done: true, .. }) => break Ok(()),
+
+                // Retry
+                Ok(WriteToIoReturn { done: false, .. }) => continue,
+                Err((_, K::Interrupted)) => continue,
+
+                Err((e, _)) => break Err(e),
+            }
+        }
+    }
+
+    pub fn write_to_stdio<W: io::Write>(&mut self, mut w: W) -> io::Result<WriteToIoReturn> {
+        if self.exhausted() {
+            return Ok(WriteToIoReturn {
+                bytes_written: 0,
+                done: true,
+            });
+        }
+
+        let buf = self.next_slice_to_write();
+        let bytes_written = w.write(buf)?;
+        self.advance(bytes_written).unwrap();
+
+        let done = self.exhausted();
+        Ok(WriteToIoReturn {
+            bytes_written,
+            done,
+        })
+    }
+
+    pub fn restart_write(&mut self) {
+        self.set_writing_position(0).unwrap()
+    }
+
+    pub fn restart_write_with_new_message(
+        &mut self,
+        len: usize,
+    ) -> Result<(), MessageLenSanityError> {
+        self.set_msg_len_and_position(len, 0)
+            .map_err(|e| e.try_into().unwrap())
+    }
+
+    pub fn next_slice_to_write(&self) -> &[u8] {
+        let s = self.header_left();
+        if !s.is_empty() {
+            return s;
+        }
+
+        let s = self.message_left();
+        if !s.is_empty() {
+            return s;
+        }
+
+        &[]
+    }
+
+    pub fn exhausted(&self) -> bool {
+        self.next_slice_to_write().is_empty()
+    }
+
+    pub fn message(&self) -> &[u8] {
+        &self.buffer_bytes()[..self.message_len()]
+    }
+
+    pub fn header_written(&self) -> &[u8] {
+        &self.header()[..self.header_offset()]
+    }
+
+    pub fn header_left(&self) -> &[u8] {
+        &self.header()[self.header_offset()..]
+    }
+
+    pub fn message_written(&self) -> &[u8] {
+        &self.message()[..self.message_offset()]
+    }
+
+    pub fn message_left(&self) -> &[u8] {
+        &self.message()[self.message_offset()..]
+    }
+
+    pub fn buf(&self) -> &Buf {
+        &self.buf
+    }
+
+    pub fn buffer_bytes(&self) -> &[u8] {
+        self.buf().borrow()
+    }
+
+    pub fn decode_header(&self) -> u64 {
+        u64::from_le_bytes(self.header)
+    }
+
+    pub fn header(&self) -> &[u8; HEADER_SIZE] {
+        &self.header
+    }
+
+    pub fn message_len(&self) -> usize {
+        self.decode_header() as usize
+    }
+
+    pub fn encoded_message_bytes(&self) -> usize {
+        self.message_len() + HEADER_SIZE
+    }
+
+    pub fn writing_position(&self) -> usize {
+        self.pos
+    }
+
+    pub fn header_offset(&self) -> usize {
+        min(self.writing_position(), HEADER_SIZE)
+    }
+
+    pub fn message_offset(&self) -> usize {
+        self.writing_position().saturating_sub(HEADER_SIZE)
+    }
+
+    pub fn set_header(&mut self, header: [u8; HEADER_SIZE]) -> Result<(), MessageLenSanityError> {
+        self.offset_transaction(|t| {
+            t.header = header;
+            t.ensure_msg_in_buf_bounds()?;
+            t.ensure_pos_in_msg_bounds()?;
+            Ok(())
+        })
+    }
+
+    pub fn encode_and_set_header(&mut self, header: u64) -> Result<(), MessageLenSanityError> {
+        self.set_header(header.to_le_bytes())
+    }
+
+    pub fn set_message_len(&mut self, len: usize) -> Result<(), MessageLenSanityError> {
+        self.encode_and_set_header(len as u64)
+    }
+
+    pub fn set_writing_position(&mut self, pos: usize) -> Result<(), PositionSanityError> {
+        self.offset_transaction(|t| {
+            t.pos = pos;
+            t.ensure_pos_in_buf_bounds()?;
+            t.ensure_pos_in_msg_bounds()?;
+            Ok(())
+        })
+    }
+
+    pub fn set_header_offset(&mut self, off: usize) -> Result<(), PositionOutOfHeaderBounds> {
+        ensure_or(off <= HEADER_SIZE, PositionOutOfHeaderBounds)?;
+        self.set_writing_position(off).unwrap();
+        Ok(())
+    }
+
+    pub fn set_message_offset(&mut self, off: usize) -> Result<(), PositionSanityError> {
+        self.set_writing_position(off + HEADER_SIZE)
+    }
+
+    pub fn advance(&mut self, off: usize) -> Result<(), PositionSanityError> {
+        self.set_writing_position(self.writing_position() + off)
+    }
+
+    pub fn set_msg_len_and_position(&mut self, len: usize, pos: usize) -> Result<(), SanityError> {
+        self.pos = 0;
+        self.set_message_len(len)?;
+        self.set_writing_position(pos)?;
+        Ok(())
+    }
+
+    fn offset_transaction<E, F>(&mut self, f: F) -> Result<(), E>
+    where
+        F: FnOnce(&mut LengthPrefixEncoder<&[u8]>) -> Result<(), E>,
+    {
+        let (header, pos) = {
+            let (buf, header, pos) = (self.buffer_bytes(), self.header, self.pos);
+            let mut tmp = LengthPrefixEncoder { buf, header, pos };
+            f(&mut tmp)?;
+            Ok((tmp.header, tmp.pos))
+        }?;
+        (self.header, self.pos) = (header, pos);
+        Ok(())
+    }
+
+    fn ensure_pos_in_buf_bounds(&self) -> Result<(), PositionOutOfBufferBounds> {
+        ensure_or(
+            self.message_offset() <= self.buffer_bytes().len(),
+            PositionOutOfBufferBounds,
+        )
+    }
+
+    fn ensure_pos_in_msg_bounds(&self) -> Result<(), PositionOutOfMessageBounds> {
+        ensure_or(
+            self.message_offset() <= self.message_len(),
+            PositionOutOfMessageBounds,
+        )
+    }
+
+    fn ensure_msg_in_buf_bounds(&self) -> Result<(), MessageTooLarge> {
+        ensure_or(
+            self.message_len() <= self.buffer_bytes().len(),
+            MessageTooLarge,
+        )
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> LengthPrefixEncoder<Buf> {
+    pub fn buf_mut(&mut self) -> &mut Buf {
+        &mut self.buf
+    }
+
+    pub fn buffer_bytes_mut(&mut self) -> &mut [u8] {
+        self.buf.borrow_mut()
+    }
+
+    pub fn message_mut(&mut self) -> &mut [u8] {
+        let off = self.message_len();
+        &mut self.buffer_bytes_mut()[..off]
+    }
+
+    pub fn message_written_mut(&mut self) -> &mut [u8] {
+        let off = self.message_offset();
+        &mut self.message_mut()[..off]
+    }
+
+    pub fn message_left_mut(&mut self) -> &mut [u8] {
+        let off = self.message_offset();
+        &mut self.message_mut()[off..]
+    }
+}
+
+impl<Buf: BorrowMut<[u8]>> Zeroize for LengthPrefixEncoder<Buf> {
+    fn zeroize(&mut self) {
+        self.buffer_bytes_mut().zeroize();
+        self.header.zeroize();
+        self.pos.zeroize();
+        self.clear();
+    }
+}

util/src/length_prefix_encoding/mod.rs

@@ -0,0 +1,2 @@
+pub mod decoder;
+pub mod encoder;

util/src/lib.rs

@@ -1,11 +1,18 @@
 #![recursion_limit = "256"]
 
 pub mod b64;
+pub mod build;
+pub mod controlflow;
 pub mod fd;
 pub mod file;
 pub mod functional;
+pub mod io;
+pub mod length_prefix_encoding;
 pub mod mem;
-pub mod ord;
+pub mod mio;
+pub mod option;
 pub mod result;
 pub mod time;
 pub mod typenum;
+pub mod zerocopy;
+pub mod zeroize;

util/src/mem.rs

@@ -1,5 +1,7 @@
 use std::borrow::{Borrow, BorrowMut};
 use std::cmp::min;
+use std::mem::{forget, swap};
+use std::ops::{Deref, DerefMut};
 
 /// Concatenate two byte arrays
 // TODO: Zeroize result?
@@ -31,3 +33,120 @@ pub fn cpy_min<T: BorrowMut<[u8]> + ?Sized, F: Borrow<[u8]> + ?Sized>(src: &F, d
     let len = min(src.len(), dst.len());
     dst[..len].copy_from_slice(&src[..len]);
 }
+
+/// Wrapper type to inhibit calling [std::mem::Drop] when the underlying variable is freed
+#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Default)]
+pub struct Forgetting<T> {
+    value: Option<T>,
+}
+
+impl<T> Forgetting<T> {
+    pub fn new(value: T) -> Self {
+        let value = Some(value);
+        Self { value }
+    }
+
+    pub fn extract(mut self) -> T {
+        let mut value = None;
+        swap(&mut value, &mut self.value);
+        value.unwrap()
+    }
+}
+
+impl<T> From<T> for Forgetting<T> {
+    fn from(value: T) -> Self {
+        Self::new(value)
+    }
+}
+
+impl<T> Deref for Forgetting<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value.as_ref().unwrap()
+    }
+}
+
+impl<T> DerefMut for Forgetting<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value.as_mut().unwrap()
+    }
+}
+
+impl<T> Borrow<T> for Forgetting<T> {
+    fn borrow(&self) -> &T {
+        self.deref()
+    }
+}
+
+impl<T> BorrowMut<T> for Forgetting<T> {
+    fn borrow_mut(&mut self) -> &mut T {
+        self.deref_mut()
+    }
+}
+
+impl<T> Drop for Forgetting<T> {
+    fn drop(&mut self) {
+        let mut value = None;
+        swap(&mut self.value, &mut value);
+        forget(value)
+    }
+}
+
+pub trait DiscardResultExt {
+    fn discard_result(self);
+}
+
+impl<T> DiscardResultExt for T {
+    fn discard_result(self) {}
+}
+
+pub trait ForgetExt {
+    fn forget(self);
+}
+
+impl<T> ForgetExt for T {
+    fn forget(self) {
+        std::mem::forget(self)
+    }
+}
+
+pub trait SwapWithExt {
+    fn swap_with(&mut self, other: Self) -> Self;
+    fn swap_with_mut(&mut self, other: &mut Self);
+}
+
+impl<T> SwapWithExt for T {
+    fn swap_with(&mut self, mut other: Self) -> Self {
+        self.swap_with_mut(&mut other);
+        other
+    }
+
+    fn swap_with_mut(&mut self, other: &mut Self) {
+        std::mem::swap(self, other)
+    }
+}
+
+pub trait SwapWithDefaultExt {
+    fn swap_with_default(&mut self) -> Self;
+}
+
+impl<T: Default> SwapWithDefaultExt for T {
+    fn swap_with_default(&mut self) -> Self {
+        self.swap_with(Self::default())
+    }
+}
+
+pub trait MoveExt {
+    /// Deliberately move the value
+    ///
+    /// Usually employed to enforce an object being
+    /// dropped after use.
+    fn move_here(self) -> Self;
+}
+
+impl<T: Sized> MoveExt for T {
+    fn move_here(self) -> Self {
+        self
+    }
+}

util/src/mio/mio.rs

@@ -0,0 +1,53 @@
+use mio::net::{UnixListener, UnixStream};
+use rustix::fd::{OwnedFd, RawFd};
+
+use crate::{
+    fd::{claim_fd, claim_fd_inplace},
+    result::OkExt,
+};
+
+pub mod interest {
+    use mio::Interest;
+    pub const R: Interest = Interest::READABLE;
+    pub const W: Interest = Interest::WRITABLE;
+    pub const RW: Interest = R.add(W);
+}
+
+pub trait UnixListenerExt: Sized {
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self>;
+}
+
+impl UnixListenerExt for UnixListener {
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self> {
+        use std::os::unix::net::UnixListener as StdUnixListener;
+
+        let sock = StdUnixListener::from(claim_fd(fd)?);
+        sock.set_nonblocking(true)?;
+        Ok(UnixListener::from_std(sock))
+    }
+}
+
+pub trait UnixStreamExt: Sized {
+    fn from_fd(fd: OwnedFd) -> anyhow::Result<Self>;
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self>;
+    fn claim_fd_inplace(fd: RawFd) -> anyhow::Result<Self>;
+}
+
+impl UnixStreamExt for UnixStream {
+    fn from_fd(fd: OwnedFd) -> anyhow::Result<Self> {
+        use std::os::unix::net::UnixStream as StdUnixStream;
+        #[cfg(target_os = "linux")] // TODO: We should support this on other plattforms
+        crate::fd::GetUnixSocketType::demand_unix_stream_socket(&fd)?;
+        let sock = StdUnixStream::from(fd);
+        sock.set_nonblocking(true)?;
+        UnixStream::from_std(sock).ok()
+    }
+
+    fn claim_fd(fd: RawFd) -> anyhow::Result<Self> {
+        Self::from_fd(claim_fd(fd)?)
+    }
+
+    fn claim_fd_inplace(fd: RawFd) -> anyhow::Result<Self> {
+        Self::from_fd(claim_fd_inplace(fd)?)
+    }
+}

util/src/mio/mod.rs

@@ -0,0 +1,13 @@
+#[allow(clippy::module_inception)]
+mod mio;
+pub use mio::*;
+
+#[cfg(feature = "experiment_file_descriptor_passing")]
+mod uds_send_fd;
+#[cfg(feature = "experiment_file_descriptor_passing")]
+pub use uds_send_fd::*;
+
+#[cfg(feature = "experiment_file_descriptor_passing")]
+mod uds_recv_fd;
+#[cfg(feature = "experiment_file_descriptor_passing")]
+pub use uds_recv_fd::*;

util/src/mio/uds_recv_fd.rs

@@ -0,0 +1,123 @@
+use std::{
+    borrow::{Borrow, BorrowMut},
+    collections::VecDeque,
+    io::Read,
+    marker::PhantomData,
+    os::fd::OwnedFd,
+};
+use uds::UnixStreamExt as FdPassingExt;
+
+use crate::fd::{claim_fd_inplace, IntoStdioErr};
+
+pub struct ReadWithFileDescriptors<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    socket: BorrowSock,
+    fds: BorrowFds,
+    _sock_dummy: PhantomData<Sock>,
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+    ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    pub fn new(socket: BorrowSock, fds: BorrowFds) -> Self {
+        let _sock_dummy = PhantomData;
+        Self {
+            socket,
+            fds,
+            _sock_dummy,
+        }
+    }
+
+    pub fn into_parts(self) -> (BorrowSock, BorrowFds) {
+        let Self { socket, fds, .. } = self;
+        (socket, fds)
+    }
+
+    pub fn socket(&self) -> &Sock {
+        self.socket.borrow()
+    }
+
+    pub fn fds(&self) -> &VecDeque<OwnedFd> {
+        self.fds.borrow()
+    }
+
+    pub fn fds_mut(&mut self) -> &mut VecDeque<OwnedFd> {
+        self.fds.borrow_mut()
+    }
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds>
+    ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: BorrowMut<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    pub fn socket_mut(&mut self) -> &mut Sock {
+        self.socket.borrow_mut()
+    }
+}
+
+impl<const MAX_FDS: usize, Sock, BorrowSock, BorrowFds> Read
+    for ReadWithFileDescriptors<MAX_FDS, Sock, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<OwnedFd>>,
+{
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        // Calculate space for additional file descriptors
+        let have_fds_before_read = self.fds().len();
+        let free_fd_slots = MAX_FDS.saturating_sub(have_fds_before_read);
+
+        // Allocate a buffer for file descriptors
+        let mut fd_buf = [0; MAX_FDS];
+        let fd_buf = &mut fd_buf[..free_fd_slots];
+
+        // Read from the unix socket
+        let (bytes_read, fds_read) = self.socket.borrow().recv_fds(buf, fd_buf)?;
+        let fd_buf = &fd_buf[..fds_read];
+
+        // Process the file descriptors
+        let mut fd_iter = fd_buf.iter();
+
+        // Try claiming all the file descriptors
+        let mut claim_fd_result = Ok(bytes_read);
+        self.fds_mut().reserve(fd_buf.len());
+        for fd in fd_iter.by_ref() {
+            match claim_fd_inplace(*fd) {
+                Ok(owned) => self.fds_mut().push_back(owned),
+                Err(e) => {
+                    // Abort on error and pass to error handler
+                    // Note that claim_fd_inplace is responsible for closing this particular
+                    // file descriptor if claiming it fails
+                    claim_fd_result = Err(e.into_stdio_err());
+                    break;
+                }
+            }
+        }
+
+        // Return if we where able to claim all file descriptors
+        if claim_fd_result.is_ok() {
+            return claim_fd_result;
+        };
+
+        // An error occurred while claiming fds
+        self.fds_mut().truncate(have_fds_before_read); // Close fds successfully claimed
+
+        // Close the remaining fds
+        for fd in fd_iter {
+            unsafe { rustix::io::close(*fd) };
+        }
+
+        claim_fd_result
+    }
+}

util/src/mio/uds_send_fd.rs

@@ -0,0 +1,121 @@
+use rustix::fd::{AsFd, AsRawFd};
+use std::{
+    borrow::{Borrow, BorrowMut},
+    cmp::min,
+    collections::VecDeque,
+    io::Write,
+    marker::PhantomData,
+};
+use uds::UnixStreamExt as FdPassingExt;
+
+use crate::{repeat, return_if};
+
+pub struct WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    socket: BorrowSock,
+    fds: BorrowFds,
+    _sock_dummy: PhantomData<Sock>,
+    _fd_dummy: PhantomData<Fd>,
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    pub fn new(socket: BorrowSock, fds: BorrowFds) -> Self {
+        let _sock_dummy = PhantomData;
+        let _fd_dummy = PhantomData;
+        Self {
+            socket,
+            fds,
+            _sock_dummy,
+            _fd_dummy,
+        }
+    }
+
+    pub fn into_parts(self) -> (BorrowSock, BorrowFds) {
+        let Self { socket, fds, .. } = self;
+        (socket, fds)
+    }
+
+    pub fn socket(&self) -> &Sock {
+        self.socket.borrow()
+    }
+
+    pub fn fds(&self) -> &VecDeque<Fd> {
+        self.fds.borrow()
+    }
+
+    pub fn fds_mut(&mut self) -> &mut VecDeque<Fd> {
+        self.fds.borrow_mut()
+    }
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: BorrowMut<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    pub fn socket_mut(&mut self) -> &mut Sock {
+        self.socket.borrow_mut()
+    }
+}
+
+impl<Sock, Fd, BorrowSock, BorrowFds> Write
+    for WriteWithFileDescriptors<Sock, Fd, BorrowSock, BorrowFds>
+where
+    Sock: FdPassingExt,
+    Fd: AsFd,
+    BorrowSock: Borrow<Sock>,
+    BorrowFds: BorrowMut<VecDeque<Fd>>,
+{
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        // At least one byte of real data should be sent when sending ancillary data. -- unix(7)
+        return_if!(buf.is_empty(), Ok(0));
+
+        // The kernel constant SCM_MAX_FD defines a limit on the number of file descriptors
+        // in the array.  Attempting to  send  an  array  larger  than  this  limit  causes
+        // sendmsg(2)  to fail with the error EINVAL.  SCM_MAX_FD has the value 253 (or 255
+        // before Linux 2.6.38).
+        // -- unix(7)
+        const SCM_MAX_FD: usize = 253;
+        let buf = match self.fds().len() <= SCM_MAX_FD {
+            false => &buf[..1], // Force caller to immediately call write() again to send its data
+            true => buf,
+        };
+
+        // Allocate the buffer for the file descriptor array
+        let fd_no = min(SCM_MAX_FD, self.fds().len());
+        let mut fd_buf = [0; SCM_MAX_FD]; // My kingdom for alloca(3)
+        let fd_buf = &mut fd_buf[..fd_no];
+
+        // Fill the file descriptor array
+        for (raw, fancy) in fd_buf.iter_mut().zip(self.fds().iter()) {
+            *raw = fancy.as_fd().as_raw_fd();
+        }
+
+        // Send data and file descriptors
+        let bytes_written = self.socket().send_fds(buf, fd_buf)?;
+
+        // Drop the file descriptors from the Deque
+        repeat!(fd_no, {
+            self.fds_mut().pop_front();
+        });
+
+        Ok(bytes_written)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}

util/src/option.rs

@@ -0,0 +1,7 @@
+pub trait SomeExt: Sized {
+    fn some(self) -> Option<Self> {
+        Some(self)
+    }
+}
+
+impl<T> SomeExt for T {}

util/src/ord.rs

@@ -1,8 +0,0 @@
-// TODO remove this once std::cmp::max becomes const
-pub const fn max_usize(a: usize, b: usize) -> usize {
-    if a > b {
-        a
-    } else {
-        b
-    }
-}

util/src/result.rs

@@ -1,5 +1,4 @@
 use std::convert::Infallible;
-use std::result::Result;
 
 /// Try block basically…returns a result and allows the use of the question mark operator inside
 #[macro_export]
@@ -9,6 +8,16 @@ macro_rules! attempt {
     };
 }
 
+pub trait OkExt<E>: Sized {
+    fn ok(self) -> Result<Self, E>;
+}
+
+impl<T, E> OkExt<E> for T {
+    fn ok(self) -> Result<Self, E> {
+        Ok(self)
+    }
+}
+
 /// Trait for container types that guarantee successful unwrapping.
 ///
 /// The `.guaranteed()` function can be used over unwrap to show that
@@ -26,6 +35,24 @@ pub trait GuaranteedValue {
     fn guaranteed(self) -> Self::Value;
 }
 
+pub trait FinallyExt {
+    fn finally<F: FnOnce(&mut Self)>(self, f: F) -> Self;
+}
+
+impl<T, E> FinallyExt for Result<T, E> {
+    fn finally<F: FnOnce(&mut Self)>(mut self, f: F) -> Self {
+        f(&mut self);
+        self
+    }
+}
+
+impl<T> FinallyExt for Option<T> {
+    fn finally<F: FnOnce(&mut Self)>(mut self, f: F) -> Self {
+        f(&mut self);
+        self
+    }
+}
+
 /// A result type that never contains an error.
 ///
 /// This is mostly useful in generic contexts.
@@ -97,3 +124,14 @@ impl<T> GuaranteedValue for Guaranteed<T> {
         self.unwrap()
     }
 }
+
+pub fn ensure_or<E>(b: bool, err: E) -> Result<(), E> {
+    match b {
+        true => Ok(()),
+        false => Err(err),
+    }
+}
+
+pub fn bail_if<E>(b: bool, err: E) -> Result<(), E> {
+    ensure_or(!b, err)
+}