From 76a813977423ab8cbcc8818bbf2be5af75a1995a Mon Sep 17 00:00:00 2001 From: G Date: Sun, 13 Sep 2026 17:30:37 +0200 Subject: [PATCH] Release 0.3.0 HTTPS-by-default for k7-api, cluster-wide Cilium isolation, first-class --docker on Kata and k7d, and RuntimeClass k7-fc. Playbook pins k7d 0.6.0. --- CHALLENGES.md | 180 ++ CHANGELOG.md | 126 ++ PERFORMANCE.md | 151 +- README.md | 87 +- ROADMAP.md | 20 +- SECURITY.md | 64 +- bench/docker-perf/README.md | 8 +- bench/docker-perf/bench.Dockerfile | 2 +- bench/docker-perf/render.py | 15 +- bench/docker-perf/results/.gitignore | 2 +- debian/changelog | 6 + docs/BACKENDS.md | 387 ++++- pyproject.toml | 4 +- setup.py | 2 +- src/k7/__init__.py | 2 +- src/k7/api/Dockerfile.api | 4 +- src/k7/api/agent.py | 4 +- src/k7/api/main.py | 22 +- src/k7/api/snapshot_gc.py | 6 +- src/k7/assets/k7-docker-cli-copy.sh | 41 + src/k7/assets/k7-docker-cli-stage.sh | 23 + src/k7/assets/k7-docker-vehicle.sh | 128 ++ src/k7/assets/k7-persist-bind.sh | 49 + src/k7/cli/_client.py | 49 +- src/k7/cli/_config.py | 14 +- src/k7/cli/k7.py | 340 +++- src/k7/core/core.py | 1491 +++++++++++++++-- src/k7/core/docker.py | 144 ++ src/k7/core/models.py | 32 +- src/k7/core/sidecar.py | 4 + src/k7/deploy/__init__.py | 1 + src/k7/deploy/inventory.ini.example | 2 +- src/k7/deploy/k7-install-node.yaml | 1115 +++++++++++- src/k7/deploy/k7_api_tls.py | 222 +++ src/k7/deploy/k7d-fc/install-firecracker.sh | 83 + src/k7/deploy/k7d-fc/pins.env | 7 + src/k7/deploy/k8s-object-presence.sh | 56 + .../manifests/k7-api/agent-daemonset.yaml | 2 +- .../{ => cilium}/agent-networkpolicy.yaml | 2 +- .../k7-api/cilium/sandbox-platform-deny.yaml | 56 + .../deploy/manifests/k7-api/clusterrole.yaml | 6 +- .../deploy/manifests/k7-api/deployment.yaml | 4 +- src/k7_sdk/__init__.py | 2 +- src/k7_sdk/client.py | 27 +- tests/conftest.py | 3 +- tests/integration/bench_backend_lifecycle.py | 19 +- tests/integration/bench_docker_perf.py | 191 ++- tests/integration/conftest.py | 84 +- tests/integration/test_api.py | 93 +- tests/integration/test_api_ingress.py | 320 ++++ tests/integration/test_cilium.py | 151 +- tests/integration/test_cluster_policy.py | 156 ++ tests/integration/test_cross_node.py | 4 +- tests/integration/test_docker.py | 671 ++++++++ tests/integration/test_firecracker.py | 31 +- tests/integration/test_k7d.py | 5 +- tests/integration/test_k7d_api.py | 56 +- tests/integration/test_k7d_fc.py | 200 +++ tests/integration/test_pause_resume.py | 14 +- tests/integration/test_qemu.py | 28 +- tests/integration/test_restore.py | 2 +- tests/integration/test_sandbox_ingress.py | 541 ++++++ tests/integration/test_sidecar_docker.py | 333 ---- tests/integration/test_snapshot_lifecycle.py | 71 +- tests/integration/test_zz_leaks.py | 16 +- tests/unit/fixtures/k7d-guest-docker-pins.env | 26 + tests/unit/test_api_auth.py | 34 +- tests/unit/test_api_lifecycle_endpoints.py | 4 +- tests/unit/test_cli_api_keys.py | 2 +- tests/unit/test_cli_api_lifecycle.py | 46 +- tests/unit/test_cli_docker.py | 79 + tests/unit/test_cli_ingress.py | 100 ++ tests/unit/test_cli_install_api_allowlist.py | 103 ++ tests/unit/test_cli_install_api_tls.py | 239 +++ tests/unit/test_cli_pause.py | 8 +- tests/unit/test_cli_resolver.py | 58 +- tests/unit/test_cli_utils.py | 23 + tests/unit/test_client.py | 26 +- tests/unit/test_core_k7d.py | 202 ++- tests/unit/test_core_mocked.py | 341 +++- tests/unit/test_core_utils.py | 33 +- tests/unit/test_docker_pins.py | 40 + tests/unit/test_k8s_object_presence.py | 124 ++ tests/unit/test_kata_docker.py | 284 ++++ tests/unit/test_katakate_shim_deprecation.py | 2 +- tests/unit/test_playbook_guest_seccomp.py | 30 + tests/unit/test_registry_ssrf.py | 2 +- tests/unit/test_render_perf.py | 21 + tests/unit/test_restore.py | 16 +- tests/unit/test_snapshot_lifecycle.py | 87 +- tutorials/k7_hetzner_node_setup.md | 4 +- uv.lock | 2 +- 92 files changed, 9016 insertions(+), 901 deletions(-) create mode 100755 src/k7/assets/k7-docker-cli-copy.sh create mode 100644 src/k7/assets/k7-docker-cli-stage.sh create mode 100755 src/k7/assets/k7-docker-vehicle.sh mode change 100644 => 100755 src/k7/assets/k7-persist-bind.sh create mode 100644 src/k7/core/docker.py create mode 100644 src/k7/deploy/__init__.py create mode 100644 src/k7/deploy/k7_api_tls.py create mode 100755 src/k7/deploy/k7d-fc/install-firecracker.sh create mode 100644 src/k7/deploy/k7d-fc/pins.env create mode 100755 src/k7/deploy/k8s-object-presence.sh rename src/k7/deploy/manifests/k7-api/{ => cilium}/agent-networkpolicy.yaml (89%) create mode 100644 src/k7/deploy/manifests/k7-api/cilium/sandbox-platform-deny.yaml create mode 100644 tests/integration/test_api_ingress.py create mode 100644 tests/integration/test_cluster_policy.py create mode 100644 tests/integration/test_docker.py create mode 100644 tests/integration/test_k7d_fc.py create mode 100644 tests/integration/test_sandbox_ingress.py delete mode 100644 tests/integration/test_sidecar_docker.py create mode 100644 tests/unit/fixtures/k7d-guest-docker-pins.env create mode 100644 tests/unit/test_cli_docker.py create mode 100644 tests/unit/test_cli_ingress.py create mode 100644 tests/unit/test_cli_install_api_allowlist.py create mode 100644 tests/unit/test_cli_install_api_tls.py create mode 100644 tests/unit/test_docker_pins.py create mode 100644 tests/unit/test_k8s_object_presence.py create mode 100644 tests/unit/test_kata_docker.py create mode 100644 tests/unit/test_playbook_guest_seccomp.py diff --git a/CHALLENGES.md b/CHALLENGES.md index deb878a..090fbfa 100644 --- a/CHALLENGES.md +++ b/CHALLENGES.md @@ -348,3 +348,183 @@ that includes `-c` trips its self-execution guard. **Reference:** none (Typer + PyInstaller). **Time lost:** ~20 min (misread as sidecar/egress failure). + +--- + +## 13. k7d `--docker`: BuildKit TLS vs dockerd pull; `-p` is guest-host netns (spec 37a-inc2) + +**Symptom:** `docker run --rm hello-world` and `docker run alpine:3.21` +succeeded in a `--docker` k7d sandbox, but `docker build -t myapp .` +from alpine:3.21 failed with `x509: certificate signed by unknown +authority` on `auth.docker.io`, and `wget http://127.0.0.1:8080` from +`k7 exec` never saw an nginx published with `-p 8080:80`. +`K7Core.exec_command` also always reports `exit_code=0`, so a failed +`docker build` looked like a successful tag that `docker run myapp` +then could not find. + +**Root cause:** dockerd's pull path uses the payload CA bundle +(`guest/docker/payload/etc/ssl/certs/ca-certificates.crt`). Default +BuildKit metadata fetch (docker driver, buildx 0.20 / buildkit v0.18) +does not, so `docker build` / compose `build:` fail TLS while `docker +pull` works. `docker run -p 8080:80` publishes in the **guest host** +netns (k7d inc1 probes it via agent vsock wget). `k7 exec` is the CRI +container netns, so localhost:8080 is the wrong place. `exec_command` +never reads the kubectl-exec exit code. + +**Fix:** tests use `DOCKER_BUILDKIT=0 docker build` (classic builder → +dockerd pull + CA) and probe the inner nginx with `docker exec web wget +http://127.0.0.1:80`. Guest exit codes are asserted via an `__K7_EC:$?` +marker. Do not "fix" this by setting `DOCKER_BUILDKIT=0` as a product +default. + +**Reference:** k7d `crates/k7d/tests/test_docker_service.rs` +`test_docker_warm_fork_nginx_survives` (`agent_sh` wget 8080); +k7d `guest/docker/payload/etc/ssl/certs/ca-certificates.crt`. + +**Time lost:** ~40 min (first integration pass). + +--- + +## 14. k3s containerd does not stamp the runtime-handler; `k7d_version` 0.5.0 would clobber the new shim (spec 38a-inc5) + +**Symptom:** RuntimeClass `k7-fc` is not enough for the shim to see +handler `k7-fc` — this node's k3s 1.36 / containerd 2.3.3 never sets +`io.kubernetes.cri.runtime-handler` on the sandbox OCI spec (k7d +CHALLENGES #228). Also, `k7 install --backend k7d-fc` with the playbook +default `k7d_version: 0.5.0` would download the public GitHub tarball +and overwrite `/usr/local/bin/containerd-shim-k7-v1` / `k7d` with a +build that does not know ConfigPath. + +**Root cause:** ConfigPath on `runtimes.k7-fc` is the working signal. +Kata Firecracker lives at `/opt/kata/bin` (playbook pin ~v1.14); +k7d-fc installs upstream v1.16.1 at `/usr/local/bin` — different +paths, do not share binaries. `grep runtimes.k7` matches `k7-fc`. + +**Fix:** playbook writes `/etc/k7d/shim-k7-fc.toml` and a ConfigPath +`[options]` table **without** `BinaryName`. Install Firecracker via +vendored `src/k7/deploy/k7d-fc/install-firecracker.sh` (sha fatal). +Smoke-test the playbook with `--tags k7d-fc` (facts tagged `always`) +and `--k7d-artifact` pointing at a tarball built from the k7d tree +that just passed `make remote-check`, never the 0.5.0 GitHub URL. +`k7_has_k7d` is true for `k7d` **or** `k7d-fc`. The allowed-backend +`difference()` list must include `k7d-fc` or a k7d-fc-only install +fails validation before any task runs. +A fifth trap: the playbook **replaces** the whole containerd template +from the selected backends. A full `k7 install --backend k7d,k7d-fc` +on a mixed k7d-dev node would drop kata/nvidia/wasm runtime blocks. +`--tags k7d-fc` installs Firecracker + toml + RuntimeClass + labels +and does not rewrite the template (`runtimes.k7-fc` is also +self-patched by k7d's `ensure_k7_fc_runtime_registered`). + +**Reference:** k7d CHALLENGES #228, `docs/backends.md`. + +**Time lost:** named in the spec before implementation (~15 min +confirming live sandbox inspect on the node). + +--- + +## 15. CRI exec into a k7-fc guest hung; kube Ready never flipped (spec 38a-inc5 / fixed 38a-inc6) + +**Symptom:** `tests/integration/test_k7d_fc.py` created a Running +`runtimeClassName: k7-fc` pod (shim log `backend=Firecracker`) then +hung forever on `K7Core.exec_command`. `timeout 15 k3s kubectl exec +… -- echo hello` exited 124. The sandbox's exec readiness probe +(`/bin/sh -c true`, 5s) also timed out, so the container never +became Ready. + +**Root cause:** Firecracker guests are reached over the jail vsock +UDS (`FcUds`), not host `AF_VSOCK`. The daemon publishes +`guest_cid = 0` (`NO_HOST_CID`). A host that dialled CID 0 blocked +in `connect()` with no timeout, so CRI Wait never saw an exit code. +This was not ConfigPath / RuntimeClass selection (k7d #228 / #230) +and not virtiofs (k7d-fc has none). + +**Fix (inc6):** k7d refuses `AF_VSOCK` CIDs 0–2 and proves +`timeout 15 kubectl exec … -- echo hello` plus kube Ready on +`k7-fc` (`test_k7_fc_exec_probe_ready_and_native_exec`). Install that +artifact with `--k7d-artifact` (not GitHub `latest`). k7-fc tests +wait on Ready again; `test_k7d_fc_exec_echo_hello` and +`TestDockerK7dFc` exec into the guest. Ready is a valid signal. + +**Reference:** k7d CHALLENGES #228 / #230 (selection) and #232 (CID 0). + +**Time lost:** ~45 min in inc5 (pytest sat on exec until the SSH +session dropped; leftover ns `k7-test-fe13c2d7` had to be deleted +by hand). + +--- + +## 16. Kata `--docker` vehicle: persist-bind hides CLI; two-PVC fork; shareProcessNamespace (spec 22a) + +**Symptom / traps while bringing `--docker` to kfd/kql:** + +1. **kql persist-bind overlays `/usr`.** The spec mounts the docker CLI via + emptyDir `subPath` onto `/usr/local/bin/docker` and + `/usr/local/lib/docker/cli-plugins/*`. kql's persist-bind then + `mount --bind`s the PVC's `/usr` over that path, so compose/buildx + vanish. Fix: also mount the CLI emptyDir at `/run/k7/docker-cli` + (persist-bind does not overlay `/run`) and restage the canonical + paths immediately after the `/usr` bind. Vehicle path-sharing + fingerprints that staging file, not `/usr`. +2. **`hostPID` is the node, not the Kata guest.** Path sharing needs the + vehicle to see the sandbox container's rootfs. `shareProcessNamespace: + true` shares the *guest* PID namespace between the two CRI containers. + `hostPID: true` would be the Kubernetes node (and is the wrong trust + domain). +3. **Two Longhorn VolumeSnapshots are not atomic.** kql fork/restore + snapshots the root PVC and the docker-graph Block PVC separately. + Each is crash-consistent (`sync` in sandbox + vehicle first); + containerd's boltdb recovers. Do not claim cross-volume atomicity. +4. **Never fall back to virtio-fs for the graph.** CHALLENGES #10's vfs + tax and virtiofsd wedge are why the graph is `volumeDevices` + ext4 + + overlay2 or fail loud. Keep the relaxed exec-probe timings (3/15/12/4). +5. **Alpine dind `VOLUME /var/lib/docker` is virtio-fs on Kata.** Unmount + it, mkfs only when blkid is not already ext4 (busybox blkid ignores + `-o value`), then `mount -t ext4`. +6. **`mount --bind` from `/proc/PID/root` is EINVAL on Kata.** Reading + through that path works. Path sharing **symlinks** `/tmp` `/home` + `/root` `/opt` `/workspace` in the vehicle onto the sandbox rootfs so + `docker run -v /tmp/x:/x` resolves. +7. **OpenEBS LVM `GetCapacity` reports VG VFree, not thin-pool free.** + `kata-vg` is almost entirely the thin-pool plus a few GiB leftover. + With `storageCapacity: true` (chart default) the scheduler sees ~5Gi + and never binds a 20Gi `k7-docker-lvm` PVC. Disable capacity tracking; + thin LVs come from the pool (`thinProvision: yes`). +8. **Kata FC virtio-fs file `subPath` mounts are invisible to Docker + plugin discovery.** `docker compose` is not a docker command on kfd + unless compose/buildx are copied into `~/.docker/cli-plugins` from + the directory-mounted emptyDir. kql persist-bind already restages + them as regular files. + +**Reference:** spec 22a-kata-docker-vehicle; CHALLENGES #10, #13. + +**Time lost:** kql integration (vehicle CrashLoopBackOff on graph mount +and path-share). + +--- + +## 17. k7-fc `--docker` Ready wait saw OutOfcpu corpses; CRI exec after pause hung + +**Symptom (bench wait):** `test_bench_k7d_fc` at 4 CPU Guaranteed left a +Ready pod plus a pile of Failed/`OutOfcpu` siblings (`requested: 4050, +used: 9890, capacity: 12000` — the just-deleted k7d sandbox's 4 CPU +still counted). `_wait_all_containers_ready` only looked at `items[0]`, +which was a Failed pod, so the wait would have timed out at 300s while +one replica was already Ready. + +**Fix:** wait until **any Running** pod has all containers Ready (same +shape as `bench_backend_lifecycle._pod_ready`). Forked-child `docker info` +asserts overlay2 on both k7d and k7d-fc. + +**Symptom (lifecycle resume):** alpine k7d-fc pause returned in 0.18s +and k7d logged `resumed vm-… (guest_cid=0)`. `_wait_exec` after resume +never returned: kubelet `Readiness probe failed: "/bin/sh -c true" +timed out after 5s`. Create→Ready and exec *before* pause had worked +(2.14s / 0.18s). This is #15 again on the post-pause path, not a +docker-perf failure. Lifecycle was aborted; do not quote k7d-fc +resume→exec from that run. + +**Reference:** CHALLENGES #15; k7d #232. + +**Time lost:** ~15 min on the OutOfcpu wait; lifecycle resume hung until +the pytest process was killed (~8 min). diff --git a/CHANGELOG.md b/CHANGELOG.md index 88c8dea..f035ebc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,132 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.3.0] — 2026-09-12 + +HTTPS-by-default for `k7-api`, cluster-wide Cilium isolation, first-class +`--docker` on Kata and k7d, and RuntimeClass `k7-fc`. The playbook now +pins k7d **0.6.0**. + +### Changed + +- **`k7 install` serves `k7-api` over HTTPS by default**. + `/etc/k7/api_endpoint` is now `https://:31007`. A + Caddy sidecar terminates TLS; the API container and its HTTP probes + stay on `:8000`. The default cert is a playbook-minted cluster CA + (`/etc/k7/tls/ca.crt`) — Let's Encrypt cannot issue for a bare IP. + **Breaking for existing laptop clients:** `K7_API_URL=http://...` must + switch to `https://` and, on the default track, install the CA + (`k7 config set api.ca ./ca.crt`, or `--api-ca` / `K7_API_CA`). + `verify=False` on an `https://` URL is an error. Optional + `--api-hostname` (Let's Encrypt via Caddy, not Certbot) and + `--api-tls-cert`/`--api-tls-key`. `--api-insecure-http` restores + today's HTTP NodePort. Keys no longer travel in cleartext; there is + still no rate limiting. + +### Added + +- **Cluster-wide sandbox platform isolation**. A deny-only + `CiliumClusterwideNetworkPolicy` (`k7-sandbox-platform-deny`) stops every + sandbox — in every egress mode, including `--egress-open` — from reaching + the node it runs on, the other nodes, the Kubernetes API server, cloud + metadata / link-local, and the `kube-system` + `longhorn-system` pods + (CoreDNS excepted). Previously an `--egress-open` sandbox could dial the + node's SSH/kubelet/k3s ports, `10.43.0.1:443`, `k7-api` and Longhorn. + The policy sets `enableDefaultDeny: false`; on Cilium 1.19 an + `egressDeny` section alone would otherwise switch the selected sandboxes + to default-deny and break open egress. Cilium-only: `--cni flannel` + clusters do not get this isolation (documented in `SECURITY.md`). +- **Opt-in sandbox ingress** — `--ingress-port` opens TCP ports on a + sandbox and `--ingress-from` (`sandbox:` / `namespace:` / + `cidr:`) scopes who may connect, as stock v1 `NetworkPolicy` rules + (so it works on Flannel too). Default is unchanged: deny all ingress. An + opened port with no source is reachable only from sandboxes in the same + namespace; the internet needs `cidr:0.0.0.0/0` spelled out. `k7 create` + warns that a `cidr:` source does not scope in-cluster peers on Cilium + (an `ipBlock` peer is not evaluated for pod-to-pod traffic) — use + `sandbox:` / `namespace:` for that. +- **`--expose-port`** — publishes a sandbox port outside the cluster through + a `NodePort` Service with `externalTrafficPolicy: Local`, so the pod sees + the real client IP instead of a SNAT'd one. Requires a matching + `--ingress-port`, prints the resolved `http://:`, + reports NodePorts in `k7 list`, and is removed by `k7 delete`. +- **`k7 install --api-allow-cidr`** (repeatable) — restricts the `k7-api` + NodePort to the operator's source CIDRs via a `CiliumNetworkPolicy` on + the API pod (`k7-api-ingress`) plus `externalTrafficPolicy: Local` so the + pod sees the real client IP. Off by default: without the flag no policy + is created and the Service is untouched. Cilium-only (`--cni flannel` + fails at the CLI — the reserved `host` / `remote-node` / `health` / + `kube-apiserver` peers the kubelet probes need are not expressible in a + v1 `NetworkPolicy`); invalid CIDRs fail before Ansible runs, and + `0.0.0.0/0` warns. This is defence in depth for operators who know their + client CIDRs, **not** a secure API: the NodePort is HTTPS by default (see + Changed above) and there is still no rate limiting. The policy selects a + pod endpoint, never the host, so a wrong CIDR never affects SSH or the + k3s API port. +- **`k7 install --hubble`** — opt-in Cilium Hubble flow observability + (relay + pinned `hubble` CLI). Off by default; not a security control. + Combined with `--cni flannel` it fails loudly. Hubble UI is not + installed (no auth, public node IP). +- **`--docker`** — first-class Docker as a guest service. On `k7d` and + `k7d-fc` the guest agent supervises a pinned `dockerd` with overlay2 + on a per-sandbox virtio-blk disk; forks stay overlay2. On Kata (`kfd` + / `kql`) the same flag injects a privileged docker-vehicle with + overlay2 on a block disk (`kql` persists/forks the graph; `kfd` is + ephemeral). `--sidecar docker` remains a deprecated alias. +- **RuntimeClass `k7-fc`** (`--backend k7d-fc`) — Firecracker under the + jailer next to the k7d daemon, CRI exec, and overlay2 on forked + `--docker` graphs. +- Kata guest seccomp is on (`disable_guest_seccomp = false`). +- Playbook default `k7d_version` is **0.6.0** (was 0.5.0). `--docker` and + `k7-fc` need that tarball; an older k7d fails loudly. + +### Fixed + +- **API sandbox delete returned 400 on every call** because + `k7 delete` always removes the `{name}-expose` Service and the + `k7-api` ClusterRole had no `services` verbs. Kubernetes answers + 403 (not 404) when the verb is missing, so even sandboxes that + never used `--expose-port` failed to delete through the API. +- **`k7 snapshot gc` left orphan VolumeSnapshotContents behind**, so + snapshot-lifecycle tests (and operator deletes) pinned namespaces in + Terminating on `volumesnapshotcontent-bound-protection` / + `pvc-as-source-protection`. GC now reaps contents whose + VolumeSnapshot is already gone or deleting, then drops the PVC + source-protection finalizer (merge-patch — strategic-merge leaves + those CRD/PVC finalizers in place). Integration teardown calls that + same sweep instead of a kubectl jsonpath that silently missed items. + +- **`k7 install` treated a busy apiserver as "Cilium is missing"**. The + DaemonSet check used `failed_when: false` and `rc != 0` as absent, so a + brief `unable to handle the request` started `cilium install` on a + cluster that already had Cilium. Presence is now present / absent / + unknown: unknown retries, then fails loud, and never installs. The + Hubble relay check got the same split (`cilium hubble enable` + Helm-upgrades the agent — do not run it because kubectl blipped). A + reinstall no longer fails `cilium status --wait` on warnings when the + DaemonSet is Ready. +- **`GET /api/v1/nodes/storage` skipped namespace authorization** + (CWE-862 / CWE-285). A leftover from the 0.2.1 scoping fix: this + cluster-scoped route was guarded only by `verify_api_key`, so a + namespace-scoped tenant key could read cluster-wide per-node storage + topology (node names, pool utilization, per-agent error strings). The + route now calls `authorize_namespace(..., all_namespaces=True)`; + scoped keys get 403, unscoped keys are unchanged. Reported privately + by **Ahmed Ibrahim** ([@skeletonsec](https://github.com/skeletonsec)), + who held disclosure — thank you. +- `k7 install --cni flannel` failed at "Apply K7 API manifests" because the + directory contained a `CiliumNetworkPolicy` and no `cilium.io` CRD + exists on Flannel. Cilium-only manifests now live in + `manifests/k7-api/cilium/` and are applied by a separate task gated on + the CNI. +- **`k7 fork` produced a sandbox with no network policy at all** — neither + the deny-ingress `NetworkPolicy` nor the egress policy was created, so a + fork of a locked-down sandbox came up with unrestricted egress. Forks now + inherit the source's egress configuration and are rolled back if their + policy cannot be created. + ## [0.2.2] — 2026-08-18 Docs and install-path release for a two-node `apt` install that ships diff --git a/PERFORMANCE.md b/PERFORMANCE.md index b5f4ed7..08c859f 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -35,7 +35,7 @@ whose fork carries live memory. kfd has no fork by design. Docker-in-VM sidecar (`--sidecar docker`) works on all three; k7d's `hello-world` pull is the slow path (tmpfs + virtio + NAT), matching the 2026-08-10 note. -## Backend lifecycle: kql vs k7d — 2026-08-10 (spec 9a M11) +## Backend lifecycle: kql vs k7d — 2026-08-10 Hetzner AX41 dedicated node (Ryzen 5 3600, 64 GiB, NVMe), Ubuntu 24.04, kernel 6.8.0-137, k3s v1.36.3, flannel CNI, Longhorn 1.10 (r=1), k7d 0.1.0. @@ -90,7 +90,7 @@ Fork is ~3x slower than a cold create. The snapshot and clone PVC steps add ~6s, Measured via integration tests (`tests/integration/test_qemu.py`) on 2026-04-10. -## Docker workloads inside sandboxes — 2026-05-29 (spec 10b) +## Docker workloads inside sandboxes — 2026-05-29 Hetzner AX52 dedicated node (Ryzen 7 7700, 64 GiB, NVMe), Ubuntu 24.04, kernel 6.8.0-100, k3s v1.35.5, docker host 29.5.2 / sandbox 27.5.1, @@ -238,14 +238,14 @@ Raw per-leg logs and the aggregated CSV are under `bench/docker-perf/results/` (gitignored — keep them in agent or local scratch space, paste into the table above when adding a new run). -## Docker-in-VM under real Longhorn replica counts (2026-08, spec 18e/18h) +## Docker-in-VM under real Longhorn replica counts (2026-08) The "r=2 vs r=1 indistinguishable" result above is **invalid**: the old bench patched a Longhorn *setting* that only affects newly-created volumes, so both legs actually ran r=1 (CHALLENGES.md #2). `bench_docker_perf.py` now sets real per-volume replica counts (`k7-ql-r3` leg). Host / kfd / r1 / r2 -medians from the 18e HA run (5 reps); **k7-ql-r3 column re-filled in -spec 18h** after the virtiofsd wedge fix (5 reps, zero VM restarts): +medians from the HA run (5 reps); **k7-ql-r3 column re-filled later** after +the virtiofsd wedge fix (5 reps, zero VM restarts): | op | host | k7-fd | k7-ql-r1 | k7-ql-r2 | k7-ql-r3 | |---|---|---|---|---|---| @@ -254,37 +254,132 @@ spec 18h** after the virtiofsd wedge fix (5 reps, zero VM restarts): | run read (venv tree cat) | — | — | — | — | 47.7s | | run cpu (10s budget) | 10.5s | 10.4s | 41.5s | 47.8s | 56.4s | -Two takeaways (spec 18f issue 8 / 18h): +Two takeaways: - **The r1→r2 jump dominates the redundancy cost** (build 108→271s; r2→r3 adds only ~8% on build): the second replica forces synchronous cross-node writes, the third mostly parallelizes with them. Post-wedge r3 `run io` (88.3s) is higher than the old single-rep 64.5s sample — that sample was the lucky survivor of a ~50% kill rate, not a median. -- **kql-r3 IO wedge — root-caused and FIXED (spec 18g):** the "VM exec - path dies after a run_io rep" wedge was not guest memory, not dockerd, - and not Longhorn faulting — the guest was healthy (load 0.4, 1.5 GB - free, zero dirty pages, clean dmesg) at the moment of death. The killer - was the **kata shim**: kata's default virtiofsd runs with +- **kql-r3 IO wedge — root-caused and FIXED:** the "VM exec path dies + after a run_io rep" wedge was not guest memory, not dockerd, and not + Longhorn faulting — the guest was healthy (load 0.4, 1.5 GB free, zero + dirty pages, clean dmesg) at the moment of death. The killer was the + **kata shim**: kata's default virtiofsd runs with `--thread-pool-size=1`, so all virtio-fs IO (container rootfs + the - Longhorn-PVC-backed `/var/lib/docker`) serializes through one thread. - A `docker run` of the ~790 MB bench image makes vfs copy the whole - rootfs and then fsync 512 MB through that single thread against an - r=3 volume (~60–80 s saturated); any agent RPC touching virtio-fs - blocks behind it, the shim's agent health ping (`CheckRequest`) times - out, and the shim declares "Dead agent" and kills the healthy VM - (`sandbox stopped unexpectedly`, pod sandbox recreated). Repro rate was - ~50% per run_io rep. Fix: `k7 install` now sets - `virtio_fs_extra_args = ["--thread-pool-size=16", ...]` in the - kata-qemu config — 18h re-ran 5/5 `run_io` + 5/5 `run_read` with zero - VM restarts on the same r=3 volume (`run_read` median 47.7s). - Exec-probe pressure was reduced too (kubelet's default 1 s exec-probe - timeout sprayed cancelled ttrpc execs — `docker info` legitimately - takes >1 s while dockerd copies vfs layers), which cuts the - `ttrpc: received message on inactive stream` noise but was NOT - sufficient on its own. + Longhorn-PVC-backed `/var/lib/docker`) serializes through one thread. A + `docker run` of the ~790 MB bench image makes vfs copy the whole rootfs + and then fsync 512 MB through that single thread against an r=3 volume + (~60–80 s saturated); any agent RPC touching virtio-fs blocks behind + it, the shim's agent health ping (`CheckRequest`) times out, and the + shim declares "Dead agent" and kills the healthy VM (`sandbox stopped + unexpectedly`, pod sandbox recreated). Repro rate was ~50% per run_io + rep. Fix: `k7 install` now sets `virtio_fs_extra_args = + ["--thread-pool-size=16", ...]` in the kata-qemu config — we re-ran 5/5 + `run_io` + 5/5 `run_read` with zero VM restarts on the same r=3 volume + (`run_read` median 47.7s). Exec-probe pressure was reduced too + (kubelet's default 1 s exec-probe timeout sprayed cancelled ttrpc execs + — `docker info` legitimately takes >1 s while dockerd copies vfs + layers), which cuts the `ttrpc: received message on inactive stream` + noise but was NOT sufficient on its own. -## k7d docker-perf leg (2026-08-11, partial) +## k7d vs k7d-fc `--docker` overlay2 — 2026-09-10 + +Same node as the 08-10 lifecycle cut (k7-node-01 / Hetzner AX41, Ryzen 5 +3600, 64 GiB, NVMe), Ubuntu 24.04, kernel 6.8.0-139, k3s v1.36.3, k7d +0.5.0 (locally built artifact, not the public GitHub tarball of the same +version string). Guests 3Gi / 4 CPU, `DOCKER_BUILDKIT=0`, graph on +virtio-blk scratch, `Storage Driver: overlay2` on the source **and** on +the forked child. 3 runs per cell after one warm-up; median (range). +⚠ marks `(max−min)/median > 0.30`. + +```bash +K7_BENCH_ENVS=k7d,k7d-fc K7_BENCH_REPS=3 K7_BENCH_WARMUP=1 \ + K7_BENCH_OUT=/tmp/bench-out \ + uv run pytest -m bench tests/integration/bench_docker_perf.py::test_bench_k7d \ + tests/integration/bench_docker_perf.py::test_bench_k7d_fc -v -s +uv run python bench/docker-perf/render.py -i /tmp/bench-out/bench-results-*.csv \ + --title "k7d vs k7d-fc --docker overlay2" --hardware "" +``` + +| Operation | k7d | k7d-fc | +|---------------------------|-----|--------| +| pull debian:12-slim | 11.3 s (9.06 s–11.5 s) | ⚠ 8.16 s (7.24 s–11.2 s) | +| build (no-cache) | 52.0 s (49.8 s–53.2 s) | 51.6 s (50.4 s–52.5 s) | +| build (cached) | ⚠ 469 ms (269 ms–477 ms) | 470 ms (369 ms–471 ms) | +| run cpu (10s budget) | 10.8 s (10.7 s–10.8 s) | 10.7 s (10.6 s–10.7 s) | +| run io (2k small + 512 MB)| ⚠ 1.18 s (1.11 s–1.48 s) | 1.28 s (1.28 s–1.38 s) | +| run read (venv tree cat) | 774 ms (674 ms–789 ms) | 674 ms (670 ms–777 ms) | +| fork warm engine (API) | 6.23 s (n=1) | 8.35 s (n=1) | +| fork → Ready + overlay2 | 6.79 s (n=1) | 9.13 s (n=1) | + +Forked-child log line: `# storage_driver_child=Storage Driver: overlay2` on +both backends (`TestDockerK7d` / `TestDockerK7dFc` also assert this). + +k7d-fc is host-parity with k7d on the storage-bound ops (no-cache build +~52 s, `run_io` ~1.3 s, `run_read` ~0.7 s). That is the overlay2 / +virtio-blk number, not the vfs-shaped ~37 s `run_io` in the earlier +sidecar rows or the 4.24 s kql Longhorn-block overlay2 row. Warm-engine +fork is the k8s adopt path (not the ~5 ms VMM CoW floor); alpine +create→Ready / fork→exec on the same node this run were **2.15 s / +2.54 s** (k7d) and **2.14 s / 2.45 s** (k7d-fc), n=1. + +k7-fc CRI exec after pause/resume hung (readiness probe timeout, +`guest_cid=0` retained) — see CHALLENGES #17. Do not quote a +resume→exec number for k7d-fc from this cut. + +## k7d `--docker` (first-class guest service) + +`k7 create --docker --backend k7d ubuntu:24.04` stamps +`k7d.katakate.org/docker=true` (no sidecar, no privileged). Graph is a +per-sandbox virtio-blk scratch disk (overlay2), so `run io` is no longer +vfs/tmpfs-shaped. `k7 fork` of a warm engine is in-scope: the child keeps +dockerd and running inner containers. + +Reproduce: + +```bash +K7_BENCH_ENVS=k7d K7_BENCH_REPS=1 K7_BENCH_WARMUP=0 \ + uv run pytest -m bench tests/integration/bench_docker_perf.py::test_bench_k7d -v -s +``` + +1-rep log from k7-node-01 (2026-09-07, k7d + guest docker payload). **Superseded** +by the 2026-09-10 k7d vs k7d-fc table above (full pull/build/cpu/io/read +column, child overlay2). Kept as the first overlay2 `run_io` datapoint: + +| op | k7d `--docker` | notes | +|---|---|---| +| `run_io` | **1.24 s** | overlay2 on virtio-blk; 2k files + 512 MB fsync | +| `fork_warm_engine` | **6.26 s** | CoW of live dockerd (not the 100–300 ms memory-fork floor; includes k7 Deployment adopt) | + +Historical sidecar/tmpfs numbers from 2026-08-11 are in the section below. + +## kql `--docker` (overlay2 on Longhorn block) + +`k7 create --docker --backend kql ubuntu:24.04` injects a privileged +`docker-vehicle` and a second Longhorn **Block** PVC (`-docker-lh`). +The graph is ext4 on that device; dockerd is forced to overlay2. This is +the number that used to be vfs-shaped (~37 s `run_io` in the earlier +sidecar/subPath rows above). Do not treat a vfs `run_io` as the kql +`--docker` baseline. + +Reproduce: + +```bash +K7_BENCH_ENVS=k7-ql-r1 K7_BENCH_REPS=1 K7_BENCH_WARMUP=0 \ + uv run pytest -m bench tests/integration/bench_docker_perf.py::test_bench_k7_ql_r1 -v -s +``` + +Numbers from this increment ( / k7-node-01, 2026-09-10, +`K7_BENCH_ENVS=k7-ql-r1 K7_BENCH_REPS=1 K7_BENCH_WARMUP=0`, +`# docker_data_path=longhorn-block-overlay2`, +`Storage Driver: overlay2`, Docker 27.5.1, Longhorn r=1 on one node): + +| op | kql `--docker` overlay2 / Longhorn block | notes | +|---|---|---| +| `run_io` | **4.24 s** | 2k files + 512 MB `dd conv=fsync`; overlay2 on Block PVC (not the ~37 s vfs row) | + +## k7d docker-perf leg (2026-08-11, partial, **sidecar / tmpfs**, superseded) `test_bench_k7d` is wired (`K7_BENCH_ENVS=k7d`). Guest sized at **3Gi / 4 vCPU** — largest size that reliably reaches Ready on this k7d build; diff --git a/README.md b/README.md index 4a42d47..da2ae29 100644 --- a/README.md +++ b/README.md @@ -76,33 +76,36 @@ The Tech Stack Sandbox backends -`k7 install --backend ` provisions one or more backends per node; `k7 create --backend …` picks one per sandbox. See [docs/BACKENDS.md](docs/BACKENDS.md) for the architecture and [PERFORMANCE.md](PERFORMANCE.md) for the full measurements (Hetzner AX41 node, medians). +`k7 install --backend ` provisions one or more backends per node; `k7 create --backend …` picks one per sandbox. See [docs/BACKENDS.md](docs/BACKENDS.md) for the architecture and [PERFORMANCE.md](PERFORMANCE.md) for the full measurements (Hetzner AX41 node, medians). -| | `kfd` (kata-firecracker-devmapper) | `kql` (kata-qemu-longhorn) | `k7d` | -|---|---|---|---| -| VMM | Firecracker (Kata) | QEMU (Kata) | k7d (custom KVM VMM) | -| RuntimeClass | `kata` | `kata-qemu` | `k7` | -| Sandbox storage | devmapper thin-pool (needs a spare raw disk) | Longhorn PVC (replicated, persistent) | erofs images + reflink XFS + guest tmpfs | -| Create → Ready* | not re-measured† | 17.1s | **2.1s** | -| Named snapshot* | — | 6.5s (Longhorn, disk-only) | — (VM snapshot trees via the k7d API) | -| Fork → usable* | — | 46.7s (disk clone + cold boot) | **~5 ms VM CoW fork**; **~2.4 s** end-to-end via k7/k8s (pod Ready + exec) | -| Pause / resume* | scale to 0 / 1 | 1.3s / 4.1s (disk survives) | **0.2s / 0.3s (VM frozen in place, memory survives)** | -| Docker-in-VM sidecar | ✅ (ephemeral docker data) | ✅ (persistent docker data; fastest `docker pull`) | ✅ (VM-lifetime docker data) | -| Cross-pod persistence | ✗ | ✅ snapshots/restore | ✗ (fork carries state instead) | +| | `kfd` (kata-firecracker-devmapper) | `kql` (kata-qemu-longhorn) | `k7d` | `k7d-fc` | +|---|---|---|---|---| +| VMM | Firecracker (Kata) | QEMU (Kata) | k7d (custom KVM VMM) | k7d + Firecracker jailer | +| RuntimeClass | `kata` | `kata-qemu` | `k7` | `k7-fc` | +| Sandbox storage | devmapper thin-pool (needs a spare raw disk) | Longhorn PVC (replicated, persistent) | erofs images + reflink XFS + guest tmpfs | same as k7d (no virtiofs / hostPath) | +| Create → Ready* | not re-measured† | 17.1s | **2.1s** | **2.1s** | +| Named snapshot* | — | 6.5s (Longhorn, disk-only) | — (VM snapshot trees via the k7d API) | — (same as k7d) | +| Fork → usable* | n/a (rejected) | 46.7s (disk clone + cold boot) | **~5 ms VM CoW fork**; **~2.4 s** end-to-end via k7/k8s (pod Ready + exec) | **~2.5 s** end-to-end (same CoW; child `--docker` stays overlay2) | +| Pause / resume* | scale to 0 / 1 | 1.3s / 4.1s (disk survives) | **0.2s / 0.3s (VM frozen in place, memory survives)** | **0.2s / —‡** | +| Docker in the VM | **`--docker`**: vehicle + overlay2 on ephemeral LVM block | **`--docker`**: vehicle + overlay2 on Longhorn block (fork/restore) | **`--docker`**: in-guest dockerd, overlay2 on virtio-blk, **forkable** | **same guest dockerd**, overlay2, **forkable** | +| Cross-pod persistence | ✗ | ✅ snapshots/restore | ✗ (fork carries state instead) | ✗ | -\* medians of 3 on one Hetzner AX41 node — methodology, ranges, and the docker-sidecar +\* medians of 3 on one Hetzner AX41 node — methodology, ranges, and docker-in-VM numbers are in [PERFORMANCE.md](PERFORMANCE.md). -† kfd needs a spare raw disk the benchmark node didn't have; its docker-workload numbers -are in the [PERFORMANCE.md](PERFORMANCE.md) spec-10b section. +† kfd needs a spare raw disk the lifecycle-bench node didn't have; Show HN +measured kfd create→exec 3.74s and **fork n/a (rejected)**. Docker-workload +numbers for kfd are in the [PERFORMANCE.md](PERFORMANCE.md) Docker benchmark section. +‡ k7-fc VMM pause/resume returns immediately; CRI exec after resume hung on this +run (`guest_cid=0` retained — CHALLENGES #17). k7d resume→exec is **0.3s**.

Also available today

-- 🛠️ Docker build / run inside VM sandboxes (docker sidecar on kfd, kql, and k7d; see [PERFORMANCE.md](PERFORMANCE.md)) +- 🛠️ Docker build / run inside VM sandboxes: k7 create --docker --backend k7d ubuntu:24.04 (or --backend k7d-fc). In-guest dockerd, overlay2, forkable. The same --docker flag on Kata (kfd/kql) injects a privileged docker-vehicle with overlay2 on a block disk — kql persists/forks the graph, kfd is ephemeral. --sidecar docker is a deprecated alias. See [PERFORMANCE.md](PERFORMANCE.md) - ⚡ Warm VM fork on the k7d backend: k7 fork CoW-clones a running sandbox's disk and memory in ~5 ms at the VMM; end-to-end through k7/Kubernetes is ~2 s to a Ready pod - 🌐 Multi-node clusters (Ansible + Longhorn) -- 🔍 Cilium CNI with FQDN egress policies +- 🔍 Cilium CNI with FQDN egress policies (optional Hubble flow observability via k7 install --hubble; off by default, observability only) - 📸 Pause / resume / fork / restore and k7 snapshot lifecycle - 🐍 Python SDK: pip install k7-sdk (katakate package deprecated) @@ -190,7 +193,7 @@ sudo apt install k7 Then let `k7` get your node ready with everything: ```console -$ k7 install --backend kfd,kql,k7d --k7d-version 0.2.1 +$ k7 install --backend kfd,kql,k7d Current task: Reminder about logging out and back in for group changes Installing K7 on 1 host(s)... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:01:41 ✅ Installation completed successfully! @@ -206,18 +209,48 @@ Optionally pass `-v` for a verbose output. > [tutorials/k7_hetzner_node_setup.md](tutorials/k7_hetzner_node_setup.md) > (this file is also in public [Katakate/k7](https://github.com/Katakate/k7)). > -> `k7` 0.2.1 from the PPA still defaults the k7d artifact to 0.1.0. Pass -> `--k7d-version 0.2.1` so install pulls the current -> [Katakate/k7d](https://github.com/Katakate/k7d/releases/tag/v0.2.1) release. -> Multi-node inventory shapes (2-node server+agent, 3-node `--ha`) are in +> The playbook pins k7d **0.6.0**. `--docker` needs the guest docker +> service (payload on the node); an older k7d fails loudly with +> `this k7d has no docker service; upgrade`. Multi-node inventory shapes +> (2-node server+agent, 3-node `--ha`) are in > `src/k7/deploy/inventory.ini.example`. +`k7 install` serves `k7-api` on NodePort `31007` over HTTPS. The default +is a playbook-minted cluster CA (Let's Encrypt cannot issue for a bare +IP). Copy `/etc/k7/tls/ca.crt` off the node and point the CLI at it: + +```bash +scp root@:/etc/k7/tls/ca.crt ./k7-ca.crt +k7 config set api.url https://:31007 +k7 config set api.ca ./k7-ca.crt +k7 config set api.key +``` + +`--api-hostname ` uses Let's Encrypt via a Caddy sidecar (the DNS +A record must point at the first master). `--api-tls-cert` + +`--api-tls-key` installs an operator-supplied pair. `--api-insecure-http` +is today's plain HTTP NodePort and must be called what it is: keys travel +in cleartext. None of this is rate limiting; do not write "the API is +now secure". + +Pass `--api-allow-cidr ` (repeatable, Cilium only) to restrict who +can connect. Off by default. Defence in depth for operators who know +their client CIDRs — it stacks with TLS and does not replace it. See +[docs/BACKENDS.md](docs/BACKENDS.md) "TLS for `k7-api`" and +"Restricting who can reach `k7-api`". + +Pass `--hubble` to turn on Cilium Hubble (relay + CLI, no UI) so policy +drops are a `hubble observe` question. Off by default; requires the +Cilium CNI (`--hubble --cni flannel` fails loudly). See +[docs/BACKENDS.md](docs/BACKENDS.md) "Debugging policy drops". + This will install and most importantly connect together the following components (depending on `--backend`): - Kubernetes (K3s prod-ready distribution) - Kata (for container virtualization) - Firecracker + Jailer + devmapper thin-pool (`kfd`) - QEMU via Kata + Longhorn PVC-backed roots (`kql`) - k7d daemon + `containerd-shim-k7-v1` + RuntimeClass `k7` (`k7d`) +- Optional: Hubble relay + `hubble` CLI when `--hubble` is passed Careful design: config updates will not touch your existing Docker or containerd setups. We chose to use K3s' own containerd for minimal disruption. Installation may however overwrite existing installations of K3s, Kata, Firecracker, Jailer, QEMU/Kata config, or Longhorn. @@ -465,7 +498,10 @@ K7 sandboxes are hardened by default with multiple layers of security: - **Linux capabilities**: All capabilities are dropped by default (`drop: ALL`) for defense-in-depth - Only explicitly add back capabilities you need via `cap_add` parameter - `allow_privilege_escalation` is always set to `false` - - Seccomp profile: `RuntimeDefault` + - Seccomp profile: `RuntimeDefault` is applied on the sandbox container + and enforced inside the guest: on k7d, and on Kata (`kfd` / `kql`) + with `disable_guest_seccomp = false` in the playbook's Kata config + (the sandbox container's OCI seccomp reaches guest runc). - **Non-root execution**: Optionally run containers and pods as non-root user (UID 65532): - `container_non_root`: Run the main container as non-root and disable privilege escalation @@ -477,9 +513,10 @@ K7 sandboxes are hardened by default with multiple layers of security: - File-based storage with 600 permissions (`/etc/k7/api_keys.json` by default) - **Network policies**: Complete network isolation for VM sandboxes - - **Ingress isolation**: All inter-VM communication is blocked by default to prevent sandbox-to-sandbox access + - **Ingress isolation**: All inter-VM communication is blocked by default to prevent sandbox-to-sandbox access; opt in per sandbox with `--ingress-port` / `--ingress-from` (`sandbox:`, `namespace:`, `cidr:`) - **Egress lockdown**: per-sandbox allowlists — CIDRs via Kubernetes NetworkPolicy, or **FQDN / domain** allowlists via Cilium (`CiliumNetworkPolicy`; default CNI) - **DNS is blocked** when egress is locked down; only entries in `egress_whitelist` (CIDR or domain) are reachable + - **Platform isolation** (Cilium only): a cluster-wide deny policy stops sandboxes in every egress mode — including `--egress-open` — from reaching the node, other nodes, the Kubernetes API, cloud metadata and the `kube-system`/`longhorn-system` pods (CoreDNS excepted). Not applied on `--cni flannel` clusters. - Administrative access via `kubectl exec` and `k7 shell` is preserved (uses Kubernetes API, not pod networking) More security features are on the roadmap (e.g. AppArmor). @@ -492,4 +529,4 @@ More security features are on the roadmap (e.g. AppArmor). - Root `setup.py` publishes the SDK; assets under `src/k7/` belong to the Debian CLI / API image, not the PyPI wheel. - User docs: `~/docs/k7/` (Mintlify). See `docs/README.md` in this repo. - The CLI Debian package is built via `src/k7/cli/build.sh` and produces `dist/k7__amd64.deb` and `dist/k7__arm64.deb`. -- CI (tags `v*`) can publish the PyPI SDK and upload the `.deb` artifact. \ No newline at end of file +- CI (tags `v*`) can publish the PyPI SDK and upload the `.deb` artifact. diff --git a/ROADMAP.md b/ROADMAP.md index ed847c1..c74f2d8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,9 +6,9 @@ Where **K7** is headed — for contributors and operators. ## Current focus -Release engineering: apt/PPA, GHCR `k7-api`, and PyPI `k7-sdk`, after the -sibling [`Katakate/k7d`](https://github.com/Katakate/k7d) v0.1.0 artifact -exists (default install URL depends on it). +Release engineering: keep the apt/PPA, GHCR `k7-api`, and PyPI `k7-sdk` +pipelines current with `main` (the PPA is still 0.2.2; HTTPS, `--docker`, +and `k7d-fc` are unreleased). --- @@ -20,16 +20,22 @@ exists (default install URL depends on it). - [x] Snapshot lifecycle + GC; restore from VolumeSnapshot - [x] CLI → API by default; `k7 api` / `k7 dev api rebuild` - [x] Docker-in-VM sidecar + bench harness +- [x] First-class `--docker` (guest dockerd on k7d / k7d-fc; docker-vehicle on Kata) - [x] Firecracker jailer +- [x] `k7d-fc` backend (k7d driving stock Firecracker + jailer) - [x] Python SDK as **`k7-sdk`** / `k7_sdk` (`katakate` deprecated) -- [x] `k7d` backend install path (artifact URL / local override) +- [x] `k7d` backend install path (public `Katakate/k7d` GitHub Releases; playbook pin 0.6.0) +- [x] HTTPS-by-default for `k7-api` (Caddy sidecar, cluster CA) +- [x] Network security hardening: cluster-wide sandbox→platform + isolation, opt-in sandbox ingress / `--expose-port`, Hubble, and a + source-CIDR allowlist for the API NodePort. Tetragon was evaluated and + rejected --- ## Next goals -- [ ] PPA (`apt install k7`), GHCR `k7-api`, PyPI `k7-sdk` -- [ ] Default `k7d` install from public `Katakate/k7d` GitHub Releases +- [ ] PPA / GHCR / PyPI cut of the unreleased work (HTTPS, `--docker`, `k7d-fc`) - [ ] Optional macOS CLI artifacts (tarball / Homebrew) — after the above --- @@ -54,4 +60,4 @@ exists (default install URL depends on it). --- -*Last updated: August 2026* +*Last updated: September 2026* diff --git a/SECURITY.md b/SECURITY.md index 95abf81..26645d2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,6 +42,10 @@ Do **not** open a public issue for security-sensitive reports. - **`k7d`** — [Katakate/k7d](https://github.com/Katakate/k7d) (`runtimeClassName: k7`); CoW sibling-fork isolation differs — see k7d's `SECURITY.md`. + - **`k7d-fc`** (`runtimeClassName: k7-fc`) — same daemon and shim; + stock Firecracker under the stock jailer. What that buys (and does + not) is k7d's `SECURITY.md` **Firecracker profile** table. hostPath + volumes are refused before the pod is created. - Sandbox containers run as non-root with restricted capabilities on top of the VM boundary. - The control plane API uses API keys with hashed storage and expiry @@ -49,8 +53,31 @@ Do **not** open a public issue for security-sensitive reports. that file). Keys may optionally be scoped to one or more namespaces (`k7 generate-api-key -n `); absent/empty scope keeps the historical unrestricted behaviour (backward compatible). Scoped keys are enforced - on every namespace-bearing endpoint — they cannot list across all - namespaces or touch namespaces outside their list. + on every namespace-bearing and cluster-scoped control-plane route — + they cannot list across all namespaces, touch namespaces outside their + list, or read cluster-wide topology (e.g. ``GET /api/v1/nodes/storage``). +- **The control-plane API is on a public NodePort (`31007`) on a public + node, over HTTPS by default**. A Caddy sidecar terminates TLS; the API + container and its probes stay HTTP on `:8000`. The default cert is a + playbook-minted cluster CA at `/etc/k7/tls/ca.crt` (Let's Encrypt cannot + issue for a bare IP). Clients trust it with `k7 config set api.ca`, + `--api-ca`, or `K7_API_CA`. `--api-hostname` (Let's Encrypt via Caddy) + and `--api-tls-cert`/`--api-tls-key` are the other tracks; + `--api-insecure-http` restores a plain HTTP NodePort and must be treated + as such (`verify=False` on an `https://` URL is an error). `k7 install + --api-allow-cidr ` (repeatable, Cilium only) restricts *who can + reach* it: a `CiliumNetworkPolicy` on the `k7-api` pod allows the + operator's CIDRs plus the reserved `host` / `remote-node` / `health` / + `kube-apiserver` peers (the kubelet probe path), and the Service switches + to `externalTrafficPolicy: Local` so the pod sees the real client IP. + This is **defence in depth for operators who know their client CIDRs, not + a secure API**: there is still no rate limiting. The flag is off by + default and affects nothing when unset. It is a pod-level policy — SSH + and the k3s API port are never touched, so a wrong CIDR costs API access + from your workstation and nothing else (`kubectl -n kube-system delete + ciliumnetworkpolicy k7-api-ingress` undoes it). A `fromCIDR` peer is not + evaluated for Cilium-managed sources, so the allowlist constrains + external clients only; in-cluster access is governed by the pod policies. - Control-plane OCI registry inspection (used to resolve image entrypoint/cmd) rejects registry hosts that are not on an allowlist (default: `registry-1.docker.io`, `ghcr.io`, `quay.io`, `public.ecr.aws`; @@ -58,10 +85,36 @@ Do **not** open a public issue for security-sensitive reports. to loopback/private/link-local/metadata/reserved addresses. Redirect following is disabled. The previous `localhost`→`http` downgrade path has been removed. -- **Ingress** to sandboxes is denied by default (NetworkPolicy). +- **Ingress** to sandboxes is denied by default (NetworkPolicy) and opt-in + per sandbox: `--ingress-port` opens TCP ports, `--ingress-from` + (`sandbox:` / `namespace:` / `cidr:`) scopes who may connect. With no + source given, an opened port is reachable only from sandboxes in the + same namespace — never from the internet. A `cidr:` source does **not** + scope in-cluster peers on Cilium: an `ipBlock` peer is not evaluated for + pod-to-pod traffic, so such a rule leaves the opened port reachable from + any sandbox in the cluster. `k7 create` warns when one is used; scope + in-cluster access with `sandbox:` / `namespace:` instead. See + `docs/BACKENDS.md` for the measured behaviour. `--expose-port` + additionally publishes a port outside the cluster (NodePort with + `externalTrafficPolicy: Local`, so the pod sees the real client IP); it + requires a matching `--ingress-port`, and on a public node it puts the + sandbox on a public IP. A `cidr:` allowlist there does not constrain + clients on the cluster's own nodes. **Egress** is per-sandbox: open, blocked, CIDR allowlist, or **FQDN** allowlist when Cilium is the CNI (default). DNS is blocked by default when egress is locked down. +- **Sandboxes never reach the platform.** With Cilium (default) a + cluster-wide deny policy (`k7-sandbox-platform-deny`) stops every + sandbox — in **every** egress mode, including `--egress-open` — from + dialing the node it runs on, the other nodes, the Kubernetes API + (node IP or `10.43.0.1`), cloud metadata (`169.254.0.0/16`), and the + pods of `kube-system` / `longhorn-system` (`k7-api`, Longhorn, ...), + CoreDNS excepted. "Open" egress means *the internet*, not *the + cluster*. This is deny-only and composes with the per-sandbox egress + policies; it does not defend against a VM escape. **Caveat:** + `k7 install --cni flannel` clusters do **not** get this isolation + (no Cilium policy engine) — an `--egress-open` sandbox there can reach + the node and the API server. - **Multi-node** clusters are supported (Ansible inventory; Longhorn for the QEMU/`kql` path). Cilium FQDN egress applies cluster-wide. @@ -69,6 +122,11 @@ See also the docs: security model, networking, and backends comparison. ### Known limitations (pre-1.0) +- Default `k7 install` serves `k7-api` over HTTPS with a playbook-minted + cluster CA, so API keys no longer travel in cleartext. There is still + no rate limiting. `--api-insecure-http` is the escape hatch back to a + plain HTTP NodePort and must be treated as such. `--api-allow-cidr` + is defence in depth on who can connect; it is not a secure-API claim. - No rate limiting or abuse protection at the API layer yet. - API key storage is local file-backed; treat the API host as trusted. Namespace scoping is an opt-in tenancy boundary on top of that model — diff --git a/bench/docker-perf/README.md b/bench/docker-perf/README.md index bbb98a1..86010fa 100644 --- a/bench/docker-perf/README.md +++ b/bench/docker-perf/README.md @@ -1,4 +1,4 @@ -# Docker workload benchmark — Spec 10b +# Docker workload benchmark Quantifies the **storage tax** of running Docker inside a k7 sandbox vs natively on the host. Output lands in [`PERFORMANCE.md`](../../PERFORMANCE.md) @@ -10,9 +10,11 @@ cited verbatim from any blog post. | Label | What it is | |-------------|------------| | `host` | Native Docker on the Hetzner node (no k7 involved) | -| `k7-fd` | k7 sandbox, `kata-firecracker-devmapper` (kfd) backend, `--sidecar docker` (docker daemon's `/var/lib/docker` is an emptyDir) | -| `k7-ql-r1` | k7 sandbox, `kata-qemu-longhorn` (kql) backend, `--sidecar docker`, Longhorn `replicas=1` | +| `k7-fd` | k7 sandbox, `kata-firecracker-devmapper` (kfd) backend, `--docker` vehicle (overlay2 on ephemeral LVM) | +| `k7-ql-r1` | k7 sandbox, `kata-qemu-longhorn` (kql) backend, `--docker` vehicle, Longhorn `replicas=1` | | `k7-ql-r2` | Same as `k7-ql-r1` but Longhorn `replicas=2` (requires ≥ 2-node cluster) | +| `k7d` | k7 sandbox, RuntimeClass `k7`, first-class `--docker` (guest dockerd, overlay2 on virtio-blk, forkable) | +| `k7d-fc` | Same guest docker service as `k7d`, RuntimeClass `k7-fc` (Firecracker + jailer) | The four environments stack the storage path cleanly: `host` → no VM, no Longhorn. `k7-fd` → VM but no Longhorn. `k7-ql-r1` → VM + one local diff --git a/bench/docker-perf/bench.Dockerfile b/bench/docker-perf/bench.Dockerfile index 93b7146..05cea3a 100644 --- a/bench/docker-perf/bench.Dockerfile +++ b/bench/docker-perf/bench.Dockerfile @@ -1,4 +1,4 @@ -# Spec 10b workload: small but realistic. apt + pip generate thousands of +# Benchmark workload: small but realistic. apt + pip generate thousands of # small files (metadata pressure), git clone is inode/dir-creation heavy, # ``dd ... conv=fsync`` measures large-block sync write throughput. Each # layer is bounded so the whole no-cache build completes in a few minutes diff --git a/bench/docker-perf/render.py b/bench/docker-perf/render.py index db7ad7d..4cefa39 100644 --- a/bench/docker-perf/render.py +++ b/bench/docker-perf/render.py @@ -1,4 +1,4 @@ -"""CSV → Markdown renderer for the Spec 10b Docker benchmark. +"""CSV → Markdown renderer for the Docker benchmark. Input CSV (produced by ``run_all.sh`` from ``/tmp/bench-