Files
G 3c80778156 Release 0.5.0
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
2026-10-18 23:21:48 +02:00

837 lines
39 KiB
Python

"""Tenancy and quotas against the live k7-api.
Every tenant here is created by the test through the admin API and deleted
with ``k7 tenant delete --cascade`` in a ``finally``; its namespaces are then
torn down and waited on like ``test_namespace``. Tenant keys are real
``--tenant`` keys in the k7-api-keys Secret, so every request takes the same
path as a customer's: auth → tenant scope → core → admission → quota lease.
Races go straight to each k7-api pod IP so both replicas contend for the
tenant's quota Lease.
"""
from __future__ import annotations
import json
import os
import subprocess
import threading
import time
from collections import Counter
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from uuid import uuid4
import httpx as _httpx
import pytest
from k7.deploy.k7_api_keys import KEYS_SECRET, generate, mint_token
from .cluster import KUBECTL, kubectl, system_ns
from .conftest import _DEV_SH, _teardown_namespace
from .test_api import (
_api_pod_ready,
_generate_test_api_key,
_get_api_base_url,
_keys_kubectl,
_revoke_test_api_key,
_tls_verify,
_wait_key_live,
)
from .test_observability import _scrape_app, _sum
pytestmark = pytest.mark.integration
KFD, KQL = "kata-firecracker-devmapper", "kata-qemu-longhorn"
IMAGE = "alpine:3.21"
SMALL = {"cpu": "250m", "memory": "256Mi"}
TIMEOUT = 300
NODE = os.uname().nodename
TENANT_PREFIX = "it41l-"
def call(method: str, url: str, token: str, *, verify: bool | str | None = None, **kw) -> _httpx.Response:
kw.setdefault("timeout", TIMEOUT)
verify = _tls_verify() if verify is None else verify
return _httpx.request(method, url, headers={"X-API-Key": token}, verify=verify, **kw)
def body(name: str, ns: str, backend: str = KFD, **extra) -> dict:
return {"name": name, "image": IMAGE, "namespace": ns, "backend": backend, "limits": dict(SMALL), **extra}
def error(r: _httpx.Response) -> dict:
return r.json()["error"]
def assert_quota_exceeded(r: _httpx.Response, dimension: str, limit: str, usage: str, requested: str) -> None:
assert r.status_code == 403, r.text
err = error(r)
assert err["code"] == "QuotaExceeded", r.text
assert {k: err["details"][k] for k in ("dimension", "limit", "usage", "requested")} == {
"dimension": dimension,
"limit": limit,
"usage": usage,
"requested": requested,
}, r.text
def kjson(*args: str) -> dict:
return json.loads(kubectl(*args, "-o", "json", check=True).stdout)
def sandboxes_in(ns: str) -> list[str]:
items = kjson("-n", ns, "get", "deployment", "-l", "katakate.org/sandbox")["items"]
return sorted(d["metadata"]["name"] for d in items)
def pvcs_in(ns: str) -> list[str]:
return sorted(p["metadata"]["name"] for p in kjson("-n", ns, "get", "pvc")["items"])
def snapshots_in(ns: str) -> list[str]:
return sorted(s["metadata"]["name"] for s in kjson("-n", ns, "get", "volumesnapshot")["items"])
def quota_leases() -> list[str]:
items = kjson("-n", system_ns(), "get", "lease")["items"]
return [n for n in (i["metadata"]["name"] for i in items) if n.startswith("k7-quota-")]
@dataclass
class Tenant:
name: str
namespaces: list[str]
token: str
base: str
@property
def ns(self) -> str:
return self.namespaces[0]
def __call__(self, method: str, path: str, **kw) -> _httpx.Response:
return call(method, f"{self.base}{path}", self.token, **kw)
def create(self, name: str, ns: str | None = None, backend: str = KFD, **extra) -> _httpx.Response:
return self("POST", "/api/v1/sandboxes", json=body(name, ns or self.ns, backend, **extra))
def wait_ready(self, name: str, ns: str | None = None, timeout: int = 300) -> None:
deadline = time.time() + timeout
while time.time() < deadline:
r = self("GET", f"/api/v1/sandboxes/{name}", params={"namespace": ns or self.ns}, timeout=10)
if r.status_code == 200 and r.json()["data"].get("ready") == "True":
return
time.sleep(2)
raise TimeoutError(f"sandbox {ns or self.ns}/{name} not ready within {timeout}s")
def snapshot(self, sandbox: str, snap: str, ns: str | None = None) -> _httpx.Response:
return self(
"POST", f"/api/v1/sandboxes/{sandbox}/snapshot", json={"snapshot_name": snap, "namespace": ns or self.ns}
)
def wait_snapshot_ready(self, snap: str, ns: str | None = None, timeout: int = 300) -> None:
deadline = time.time() + timeout
while time.time() < deadline:
r = self("GET", f"/api/v1/snapshots/{snap}", params={"namespace": ns or self.ns}, timeout=10)
if r.status_code == 200 and r.json()["data"].get("ready_to_use"):
return
time.sleep(2)
raise TimeoutError(f"snapshot {ns or self.ns}/{snap} not ready within {timeout}s")
def _wait_api(url: str) -> None:
for _ in range(30):
try:
if _httpx.get(f"{url}/health", verify=_tls_verify(), timeout=3).status_code == 200:
return
except _httpx.HTTPError:
pass
time.sleep(2)
pytest.fail(f"k7-api {url}/health not reachable", pytrace=False)
@pytest.fixture(scope="module")
def base() -> str:
if not _api_pod_ready():
pytest.fail("k7-api deployment not ready; tenancy tests need the API", pytrace=False)
url = _get_api_base_url()
_wait_api(url)
return url
@pytest.fixture(scope="module")
def admin(base: str) -> Iterator[str]:
name = f"{TENANT_PREFIX}admin-{uuid4().hex[:6]}"
try:
yield _generate_test_api_key(name=name, role="admin")
finally:
_revoke_test_api_key(name)
@contextmanager
def tenant(base: str, admin_token: str, *, namespaces: int = 1, **spec) -> Iterator[Tenant]:
"""A test-owned Tenant + ``--tenant`` key; ``tenant delete --cascade`` and namespace teardown in ``finally``."""
name = f"{TENANT_PREFIX}{uuid4().hex[:6]}"
nss = [f"{name}-{i}" for i in range(namespaces)]
created = False
try:
r = call(
"POST", f"{base}/api/v1/tenants", admin_token, json={"name": name, "spec": {"namespaces": nss, **spec}}
)
assert r.status_code == 201, r.text
created = True
token, _id, _entry = generate(
_keys_kubectl(), f"{name}-key", role="user", tenant=name, expires_days=1, namespaces=[], nodes=[]
)
_wait_key_live(token)
yield Tenant(name, nss, token, base)
finally:
failures = []
if created and kubectl("get", "tenant", name).returncode == 0:
out = subprocess.run(
[str(_DEV_SH), "tenant", "delete", name, "--cascade"], capture_output=True, text=True, timeout=900
)
if out.returncode != 0:
failures.append(f"tenant delete {name} --cascade: {out.stderr or out.stdout}")
_revoke_test_api_key(f"{name}-key")
for ns in nss:
if kubectl("get", "namespace", ns).returncode == 0:
_teardown_namespace(ns)
assert not failures, failures
@pytest.fixture()
def make_tenant(base: str, admin: str):
with ExitStack() as stack:
yield lambda **kw: stack.enter_context(tenant(base, admin, **kw))
def api_pod_urls() -> tuple[list[str], bool | str]:
"""Each k7-api replica's own URL, and ``verify``: Caddy's TLS port when the sidecar runs, else :8000."""
pods = kjson("-n", system_ns(), "get", "pods", "-l", "app=k7-api")["items"]
urls = []
for p in pods:
if p["status"].get("phase") != "Running" or not p["status"].get("podIP"):
continue
caddy = next((c for c in p["spec"]["containers"] if c["name"] == "caddy"), None)
ip = p["status"]["podIP"]
urls.append(f"https://{ip}:{caddy['ports'][0]['containerPort']}" if caddy else f"http://{ip}:8000")
assert len(urls) >= 2, f"the race needs both k7-api replicas, found {urls}"
# Pod IPs are not in the API certificate's SANs: TLS to Caddy, unverified.
return urls, False
def race(t: Tenant, n: int, request) -> Counter:
"""``n`` requests released together, alternating between the k7-api pods; status code counts."""
urls, verify = api_pod_urls()
gate = threading.Barrier(n)
def one(i: int) -> tuple[int, str]:
method, path, payload = request(i)
gate.wait()
r = call(method, f"{urls[i % len(urls)]}{path}", t.token, verify=verify, json=payload)
return r.status_code, r.text
with ThreadPoolExecutor(n) as pool:
results = list(pool.map(one, range(n)))
codes = Counter(code for code, _ in results)
unexpected = [text for code, text in results if code not in (201, 403)]
assert not unexpected, f"race returned non-201/403: {unexpected[:3]}"
for code, text in results:
if code == 403:
assert json.loads(text)["error"]["code"] == "QuotaExceeded", text
assert quota_leases() == [], "quota leases left behind after the race"
return codes
# ── A: scope isolation across sandboxes, snapshots, forks and restore ──
@pytest.fixture(scope="module")
def pair(base: str, admin: str) -> Iterator[tuple[Tenant, Tenant]]:
"""Tenant A (two namespaces, a running kql sandbox ``src`` and a snapshot ``src-snap``) and tenant B."""
with tenant(base, admin, namespaces=2) as a, tenant(base, admin) as b:
r = a.create("src", backend=KQL, root_disk_size="2Gi")
assert r.status_code == 201, r.text
a.wait_ready("src")
r = a.snapshot("src", "src-snap")
assert r.status_code == 201, r.text
a.wait_snapshot_ready("src-snap")
yield a, b
def _routes(ns: str) -> list[tuple[str, str, str, dict]]:
"""(label, method, path, kwargs) for every namespaced route, aimed at A's ``src`` / ``src-snap`` in ``ns``."""
q = {"params": {"namespace": ns}}
return [
("list sandboxes", "GET", "/api/v1/sandboxes", q),
("get sandbox", "GET", "/api/v1/sandboxes/src", q),
("logs", "GET", "/api/v1/sandboxes/src/logs", q),
("exec", "POST", "/api/v1/sandboxes/src/exec", {**q, "json": {"command": "id"}}),
("create sandbox", "POST", "/api/v1/sandboxes", {"json": body("intruder", ns)}),
("pause", "POST", "/api/v1/sandboxes/src/pause", {"json": {"namespace": ns}}),
("pause+snapshot", "POST", "/api/v1/sandboxes/src/pause", {"json": {"namespace": ns, "snapshot": "x"}}),
("resume", "POST", "/api/v1/sandboxes/src/resume", {"json": {"namespace": ns}}),
("fork", "POST", "/api/v1/sandboxes/src/fork", {"json": {"new_name": "intruder", "namespace": ns}}),
(
"snapshot create",
"POST",
"/api/v1/sandboxes/src/snapshot",
{"json": {"snapshot_name": "x", "namespace": ns}},
),
("snapshot list", "GET", "/api/v1/snapshots", q),
("snapshot get", "GET", "/api/v1/snapshots/src-snap", q),
(
"restore",
"POST",
"/api/v1/snapshots/src-snap/restore",
{"json": {"new_sandbox_name": "intruder", "namespace": ns}},
),
("snapshot gc", "POST", "/api/v1/snapshots/gc", {"json": {"namespace": ns, "keep_fork_for": "0s"}}),
("snapshot delete", "DELETE", "/api/v1/snapshots/src-snap", q),
("delete sandbox", "DELETE", "/api/v1/sandboxes/src", q),
("delete all sandboxes", "DELETE", "/api/v1/sandboxes", q),
]
class TestTenantIsolation:
def test_cross_tenant_read_write_delete_refused(self, pair):
"""B's key gets 403 on every route in A's namespaces, and A's objects are untouched."""
a, b = pair
before = (sandboxes_in(a.ns), pvcs_in(a.ns), snapshots_in(a.ns))
for ns in a.namespaces:
for label, method, path, kw in _routes(ns):
r = b(method, path, **kw)
assert r.status_code == 403, f"{label} in {ns} with tenant B's key: {r.status_code} {r.text}"
assert (sandboxes_in(a.ns), pvcs_in(a.ns), snapshots_in(a.ns)) == before
assert sandboxes_in(a.namespaces[1]) == []
a.wait_ready("src") # never paused
def test_reverse_direction_refused(self, pair):
a, b = pair
for label, method, path, kw in _routes(b.ns):
r = a(method, path, **kw)
assert r.status_code == 403, f"{label} in {b.ns} with tenant A's key: {r.status_code} {r.text}"
assert sandboxes_in(b.ns) == []
def test_fork_and_restore_cannot_cross_into_another_tenant(self, pair):
"""Fork and restore are namespace-local: B naming A's source from B's own namespace finds nothing."""
a, b = pair
r = b("POST", "/api/v1/sandboxes/src/fork", json={"new_name": "stolen", "namespace": b.ns})
assert r.status_code == 404, r.text
r = b("POST", "/api/v1/snapshots/src-snap/restore", json={"new_sandbox_name": "stolen", "namespace": b.ns})
assert r.status_code == 404, r.text
r = b("GET", "/api/v1/snapshots/src-snap", params={"namespace": b.ns})
assert r.status_code == 404, r.text
assert sandboxes_in(b.ns) == [] and pvcs_in(b.ns) == []
def test_cluster_wide_routes_refused_for_tenant_keys(self, pair):
a, _b = pair
for method, path, kw in (
("GET", "/api/v1/sandboxes", {}),
("GET", "/api/v1/snapshots", {"params": {"all_namespaces": "true"}}),
("POST", "/api/v1/snapshots/gc", {"json": {"all_namespaces": True}}),
("GET", "/api/v1/nodes/storage", {}),
("GET", "/api/v1/tenants", {}),
("DELETE", f"/api/v1/tenants/{a.name}", {}),
):
r = a(method, path, **kw)
assert r.status_code == 403, f"{method} {path}: {r.status_code} {r.text}"
def test_tenant_key_reaches_every_tenant_namespace(self, pair):
a, _b = pair
for ns in a.namespaces:
for path in ("/api/v1/sandboxes", "/api/v1/snapshots"):
r = a("GET", path, params={"namespace": ns})
assert r.status_code == 200, f"{path} in {ns}: {r.text}"
def test_fork_and_restore_within_tenant_carry_identity(self, pair):
a, _b = pair
r = a("POST", "/api/v1/sandboxes/src/fork", json={"new_name": "src-fork", "namespace": a.ns})
assert r.status_code == 201, r.text
r = a("POST", "/api/v1/snapshots/src-snap/restore", json={"new_sandbox_name": "src-rest", "namespace": a.ns})
assert r.status_code == 201, r.text
try:
for name in ("src", "src-fork", "src-rest"):
a.wait_ready(name)
key_ids = set()
for kind, selector in (
("deployment", "katakate.org/sandbox"),
("pod", "katakate.org/sandbox"),
("pvc", "k7.katakate.org/tenant"),
("volumesnapshot", "k7.katakate.org/tenant"),
):
items = kjson("-n", a.ns, "get", kind, "-l", selector)["items"]
assert items, f"no {kind} in {a.ns}"
for obj in items:
labels = obj["metadata"].get("labels") or {}
assert labels.get("k7.katakate.org/tenant") == a.name, f"{kind} {obj['metadata']['name']}: {labels}"
assert labels.get("k7.katakate.org/api-key-id"), f"{kind} {obj['metadata']['name']}: {labels}"
key_ids.add(labels["k7.katakate.org/api-key-id"])
assert len(key_ids) == 1, f"objects carry different key ids: {key_ids}"
assert {"src-fork", "src-rest"} <= set(sandboxes_in(a.ns))
finally:
for name in ("src-fork", "src-rest"):
a("DELETE", f"/api/v1/sandboxes/{name}", params={"namespace": a.ns})
def test_new_namespace_reaches_existing_key_and_removal_revokes(self, pair, admin):
a, _b = pair
extra = f"{a.name}-added"
try:
r = call("PATCH", f"{a.base}/api/v1/tenants/{a.name}", admin, json={"namespaces": [*a.namespaces, extra]})
assert r.status_code == 200, r.text
r = a("GET", "/api/v1/sandboxes", params={"namespace": extra})
assert r.status_code == 200, f"same key, newly added namespace: {r.text}"
r = call("PATCH", f"{a.base}/api/v1/tenants/{a.name}", admin, json={"namespaces": a.namespaces})
assert r.status_code == 200, r.text
r = a("GET", "/api/v1/sandboxes", params={"namespace": extra})
assert r.status_code == 403, f"removed namespace must be refused at once: {r.text}"
finally:
if kubectl("get", "namespace", extra).returncode == 0:
_teardown_namespace(extra)
def test_double_claimed_namespace_is_500_naming_both(self, pair, admin):
a, b = pair
r = call(
"POST", f"{a.base}/api/v1/tenants", admin, json={"name": f"{a.name}-x", "spec": {"namespaces": [a.ns]}}
)
assert r.status_code == 400 and a.name in r.text, f"the API refuses a second claim: {r.text}"
# A hand-written CR (kubectl apply) bypasses that check; every request there then fails loudly.
rogue = f"{TENANT_PREFIX}rogue-{uuid4().hex[:6]}"
cr = {
"apiVersion": "k7.katakate.org/v1alpha1",
"kind": "Tenant",
"metadata": {"name": rogue},
"spec": {"namespaces": [a.ns]},
}
subprocess.run([*KUBECTL, "apply", "-f", "-"], input=json.dumps(cr), text=True, check=True)
try:
for who in (a, b):
r = who("GET", "/api/v1/sandboxes", params={"namespace": a.ns})
assert r.status_code in (403, 500), r.text
r = a("GET", "/api/v1/sandboxes", params={"namespace": a.ns})
assert r.status_code == 500, r.text
err = error(r)
assert err["code"] == "TenantConflict" and sorted(err["details"]["tenants"]) == sorted([a.name, rogue])
r = a("POST", "/api/v1/sandboxes", json=body("blocked", a.ns))
assert r.status_code == 500, r.text
finally:
kubectl("delete", "tenant", rogue, "--ignore-not-found", check=True)
assert a("GET", "/api/v1/sandboxes", params={"namespace": a.ns}).status_code == 200
def test_orphan_tenant_key_is_refused(self, base, admin):
"""A key whose tenant was deleted reaches nothing, not everything."""
with tenant(base, admin) as t:
assert call("DELETE", f"{base}/api/v1/tenants/{t.name}", admin).status_code == 200
for params in ({"namespace": t.ns}, {"namespace": "default"}, {}):
r = t("GET", "/api/v1/sandboxes", params=params)
assert r.status_code == 403 and "does not exist" in r.text, f"{params}: {r.text}"
class TestTenantNodes:
def test_sandboxes_land_on_tenant_nodes_without_a_pin(self, make_tenant):
t = make_tenant(nodes=[NODE])
node = kjson("get", "node", NODE)
assert node["metadata"]["labels"].get("k7.katakate.org/tenant") == t.name
assert any(x.get("key") == "k7.katakate.org/tenant" for x in node["spec"].get("taints") or [])
r = t.create("placed")
assert r.status_code == 201, r.text
t.wait_ready("placed")
pod = kjson("-n", t.ns, "get", "pod", "-l", "katakate.org/sandbox=placed")["items"][0]
assert pod["spec"]["nodeSelector"].get("k7.katakate.org/tenant") == t.name
assert pod["spec"]["nodeName"] == NODE
def test_tenant_delete_releases_its_nodes(self, base, admin):
with tenant(base, admin, nodes=[NODE]):
pass
node = kjson("get", "node", NODE)
assert "k7.katakate.org/tenant" not in (node["metadata"].get("labels") or {})
assert not any(x.get("key") == "k7.katakate.org/tenant" for x in node["spec"].get("taints") or [])
# ── C: quota boundaries, one dimension at a time ──
class TestQuotaBoundaries:
def test_sandboxes(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 2})
for i in range(2):
assert t.create(f"s{i}").status_code == 201
assert_quota_exceeded(t.create("s2"), "sandboxes", "2", "2", "1")
assert sandboxes_in(t.ns) == ["s0", "s1"]
rejected = _sum(_scrape_app("k7-api"), "k7_quota_rejections_total", tenant=t.name, dimension="sandboxes")
assert rejected == 1, "the refusal is counted once, on the replica that served it"
def test_sandboxes_across_namespaces(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"sandboxes": 2})
assert t.create("s0", t.namespaces[0]).status_code == 201
assert t.create("s1", t.namespaces[1]).status_code == 201
assert_quota_exceeded(t.create("s2", t.namespaces[1]), "sandboxes", "2", "2", "1")
def test_vcpu(self, make_tenant):
t = make_tenant(quotas={"vcpu": "1"})
for i in range(2):
assert t.create(f"c{i}", limits={"cpu": "500m", "memory": "256Mi"}).status_code == 201
r = t.create("c2", limits={"cpu": "500m", "memory": "256Mi"})
assert_quota_exceeded(r, "vcpu", "1", "1", "500m")
def test_vcpu_quota_needs_a_cpu_limit(self, make_tenant):
t = make_tenant(quotas={"vcpu": "1"})
r = t("POST", "/api/v1/sandboxes", json={**body("nolimit", t.ns), "limits": {"memory": "256Mi"}})
assert r.status_code == 400 and "cpu" in r.text, r.text
assert sandboxes_in(t.ns) == []
def test_memory(self, make_tenant):
t = make_tenant(quotas={"memory": "1Gi"})
for i in range(2):
assert t.create(f"m{i}", limits={"cpu": "250m", "memory": "512Mi"}).status_code == 201
r = t.create("m2", limits={"cpu": "250m", "memory": "512Mi"})
assert_quota_exceeded(r, "memory", "1Gi", "1Gi", "512Mi")
def test_disk_kfd(self, make_tenant):
"""kfd reserves its 10 GB devmapper root per sandbox."""
t = make_tenant(quotas={"disk": "25G"})
for i in range(2):
assert t.create(f"d{i}").status_code == 201
assert_quota_exceeded(t.create("d2"), "disk", "25G", "20G", "10G")
def test_disk_kql_rolls_back_the_refused_pvc(self, make_tenant):
t = make_tenant(quotas={"disk": "5Gi"})
for i in range(2):
assert t.create(f"q{i}", backend=KQL, root_disk_size="2Gi").status_code == 201
before = pvcs_in(t.ns)
r = t.create("q2", backend=KQL, root_disk_size="2Gi")
assert_quota_exceeded(r, "disk", "5Gi", "4Gi", "2Gi")
assert pvcs_in(t.ns) == before and sandboxes_in(t.ns) == ["q0", "q1"]
def test_snapshot_storage(self, make_tenant):
t = make_tenant(quotas={"snapshotStorage": "3Gi"})
assert t.create("src", backend=KQL, root_disk_size="2Gi").status_code == 201
t.wait_ready("src")
r = t.snapshot("src", "snap0")
assert r.status_code == 201, r.text
r = t.snapshot("src", "snap1")
assert_quota_exceeded(r, "snapshotStorage", "3Gi", "2Gi", "2Gi")
r = t("POST", "/api/v1/sandboxes/src/pause", json={"namespace": t.ns, "snapshot": "snap2"})
assert_quota_exceeded(r, "snapshotStorage", "3Gi", "2Gi", "2Gi")
assert snapshots_in(t.ns) == ["snap0"]
t.wait_ready("src") # the refused pause+snapshot left it running
def test_fork_over_quota_leaves_nothing_behind(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 1})
assert t.create("src", backend=KQL, root_disk_size="2Gi").status_code == 201
t.wait_ready("src")
before = (pvcs_in(t.ns), snapshots_in(t.ns))
r = t("POST", "/api/v1/sandboxes/src/fork", json={"new_name": "f", "namespace": t.ns})
assert_quota_exceeded(r, "sandboxes", "1", "1", "1")
assert (pvcs_in(t.ns), snapshots_in(t.ns)) == before and sandboxes_in(t.ns) == ["src"]
def test_restore_over_quota_leaves_nothing_behind(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 1})
assert t.create("src", backend=KQL, root_disk_size="2Gi").status_code == 201
t.wait_ready("src")
assert t.snapshot("src", "snap").status_code == 201
t.wait_snapshot_ready("snap")
before = (pvcs_in(t.ns), snapshots_in(t.ns))
r = t("POST", "/api/v1/snapshots/snap/restore", json={"new_sandbox_name": "r", "namespace": t.ns})
assert_quota_exceeded(r, "sandboxes", "1", "1", "1")
assert (pvcs_in(t.ns), snapshots_in(t.ns)) == before and sandboxes_in(t.ns) == ["src"]
def test_resume_is_not_a_create(self, make_tenant):
"""Pause keeps the reservation (reserved-not-used), so resume never trips the quota."""
t = make_tenant(quotas={"sandboxes": 1})
assert t.create("p").status_code == 201
assert t("POST", "/api/v1/sandboxes/p/pause", json={"namespace": t.ns}).status_code == 200
assert_quota_exceeded(t.create("p2"), "sandboxes", "1", "1", "1")
assert t("POST", "/api/v1/sandboxes/p/resume", json={"namespace": t.ns}).status_code == 200
def test_freeing_quota_admits_again(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 1})
assert t.create("a").status_code == 201
assert t.create("b").status_code == 403
assert t("DELETE", "/api/v1/sandboxes/a", params={"namespace": t.ns}).status_code == 200
assert t.create("b").status_code == 201, "a deleted sandbox frees its quota"
def test_max_sandbox_size_and_default_size(self, make_tenant):
t = make_tenant(defaultSize={"cpu": "500m", "memory": "512Mi"}, limits={"maxSandboxSize": {"cpu": "1"}})
r = t.create("big", limits={"cpu": "2", "memory": "256Mi"})
assert r.status_code == 400 and "maxSandboxSize.cpu" in r.text, r.text
r = t("POST", "/api/v1/sandboxes", json={"name": "dflt", "image": IMAGE, "namespace": t.ns, "backend": KFD})
assert r.status_code == 201, r.text
dep = kjson("-n", t.ns, "get", "deployment", "dflt")
lim = next(c for c in dep["spec"]["template"]["spec"]["containers"] if c["name"] == "sandbox")["resources"]
assert lim["limits"]["cpu"] == "500m" and lim["limits"]["memory"] == "512Mi", lim
def test_lease_timeout_is_503_and_nothing_created(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 5})
lease = f"k7-quota-{t.name}"
now = time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime())
held = {
"apiVersion": "coordination.k8s.io/v1",
"kind": "Lease",
"metadata": {"name": lease, "namespace": system_ns()},
"spec": {
"holderIdentity": "it41l-foreign",
"leaseDurationSeconds": 120,
"acquireTime": now,
"renewTime": now,
},
}
subprocess.run([*KUBECTL, "create", "-f", "-"], input=json.dumps(held), text=True, check=True)
try:
t0 = time.monotonic()
r = t.create("blocked")
assert r.status_code == 503 and error(r)["code"] == "QuotaLockTimeout", r.text
assert time.monotonic() - t0 >= 5, "gave up before the 5 s acquire timeout"
assert sandboxes_in(t.ns) == [] and pvcs_in(t.ns) == []
holder = kjson("-n", system_ns(), "get", "lease", lease)["spec"]["holderIdentity"]
assert holder == "it41l-foreign", "the API must never take or delete a live foreign lease"
finally:
kubectl("-n", system_ns(), "delete", "lease", lease, "--ignore-not-found", check=True)
assert t.create("after").status_code == 201
def test_status_usage_converges_and_backstop_resource_quota(self, make_tenant):
t = make_tenant(quotas={"sandboxes": 3, "disk": "50G"})
for i in range(2):
assert t.create(f"u{i}").status_code == 201
deadline, usage = time.time() + 75, {}
while time.time() < deadline:
usage = (kjson("get", "tenant", t.name).get("status") or {}).get("usage") or {}
if usage.get("sandboxes") == "2":
break
time.sleep(3)
assert usage.get("sandboxes") == "2" and usage.get("disk") == "20G", f"status.usage after 75 s: {usage}"
rq = kjson("-n", t.ns, "get", "resourcequota", "k7-tenant-quota")
assert rq["spec"]["hard"] == {"count/deployments.apps": "3", "requests.storage": "50G"}
# ── C: concurrent creates racing the quota through both API replicas ──
class TestQuotaRaces:
def _create(self, t: Tenant, backend: str = KFD, **extra):
return lambda i: (
"POST",
"/api/v1/sandboxes",
body(f"r{i}", t.namespaces[i % len(t.namespaces)], backend, **extra),
)
def _count(self, t: Tenant) -> int:
return sum(len(sandboxes_in(ns)) for ns in t.namespaces)
def test_sandboxes_20_against_10(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"sandboxes": 10})
codes = race(t, 20, self._create(t))
assert codes == Counter({201: 10, 403: 10}) and self._count(t) == 10, codes
def test_vcpu(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"vcpu": "2"})
codes = race(t, 20, self._create(t, limits={"cpu": "500m", "memory": "256Mi"}))
assert codes == Counter({201: 4, 403: 16}) and self._count(t) == 4, codes
def test_memory(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"memory": "1Gi"})
codes = race(t, 20, self._create(t, limits={"cpu": "250m", "memory": "256Mi"}))
assert codes == Counter({201: 4, 403: 16}) and self._count(t) == 4, codes
def test_disk_kfd(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"disk": "50G"})
codes = race(t, 20, self._create(t))
assert codes == Counter({201: 5, 403: 15}) and self._count(t) == 5, codes
def test_disk_kql(self, make_tenant):
t = make_tenant(namespaces=2, quotas={"disk": "10Gi"})
codes = race(t, 20, self._create(t, backend=KQL, root_disk_size="2Gi"))
assert codes == Counter({201: 5, 403: 15}) and self._count(t) == 5, codes
for ns in t.namespaces:
claimed = {
v["persistentVolumeClaim"]["claimName"]
for d in kjson("-n", ns, "get", "deployment", "-l", "katakate.org/sandbox")["items"]
for v in d["spec"]["template"]["spec"].get("volumes") or []
if "persistentVolumeClaim" in v
}
assert set(pvcs_in(ns)) == claimed, f"refused creates left PVCs in {ns}"
def test_snapshots(self, make_tenant):
t = make_tenant(quotas={"snapshotStorage": "5Gi"})
assert t.create("src", backend=KQL, root_disk_size="2Gi").status_code == 201
t.wait_ready("src")
codes = race(
t, 10, lambda i: ("POST", "/api/v1/sandboxes/src/snapshot", {"snapshot_name": f"s{i}", "namespace": t.ns})
)
assert codes == Counter({201: 2, 403: 8}) and len(snapshots_in(t.ns)) == 2, codes
# ── B: production mode, on a test-owned copy of the k7-api pod ──
@contextmanager
def _probe_pod(env: dict[str, str]) -> Iterator[str]:
"""A test-owned copy of the k7-api pod template with ``env`` overridden; yields its name."""
dep = kjson("-n", system_ns(), "get", "deployment", "k7-api")
spec = dep["spec"]["template"]["spec"]
spec.pop("affinity", None)
spec.pop("topologySpreadConstraints", None)
spec["restartPolicy"] = "Never"
for c in spec["containers"]:
if c["name"] == "k7-api":
c["env"] = [e for e in c.get("env") or [] if e["name"] not in env] + [
{"name": k, "value": v} for k, v in env.items()
]
for probe in ("readinessProbe", "livenessProbe", "startupProbe"):
c.pop(probe, None)
name = f"{TENANT_PREFIX}api-probe-{uuid4().hex[:6]}"
pod = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {"name": name, "namespace": system_ns(), "labels": {"app": "k7-api-probe"}},
"spec": spec,
}
subprocess.run([*KUBECTL, "create", "-f", "-"], input=json.dumps(pod), text=True, check=True)
try:
yield name
finally:
kubectl("-n", system_ns(), "delete", "pod", name, "--ignore-not-found", "--wait=false", check=True)
def _k7_api_terminated(name: str) -> bool:
st = kjson("-n", system_ns(), "get", "pod", name)["status"]
return any(
(s.get("state") or {}).get("terminated") for s in st.get("containerStatuses") or [] if s["name"] == "k7-api"
)
@contextmanager
def _api_probe(env: dict[str, str]) -> Iterator[str]:
"""Run the probe pod once; yield the k7-api container's logs after it exits (it must refuse to start)."""
with _probe_pod(env) as name:
deadline = time.time() + 180
while not _k7_api_terminated(name):
if time.time() > deadline:
pytest.fail(f"production-mode probe {name} is still running after 180 s; it must refuse to start")
time.sleep(2)
yield kubectl("-n", system_ns(), "logs", name, "-c", "k7-api", check=True).stdout
@contextmanager
def _running_api_probe(env: dict[str, str]) -> Iterator[str]:
"""Run the probe pod and yield its own URL once ``/health`` answers (TLS to Caddy, unverified)."""
with _probe_pod(env) as name:
deadline = time.time() + 180
while True:
if _k7_api_terminated(name):
logs = kubectl("-n", system_ns(), "logs", name, "-c", "k7-api").stdout
pytest.fail(f"probe {name} exited; it should be serving:\n{logs}")
pod = kjson("-n", system_ns(), "get", "pod", name)
caddy = next((c for c in pod["spec"]["containers"] if c["name"] == "caddy"), None)
ip = pod["status"].get("podIP")
url = f"https://{ip}:{caddy['ports'][0]['containerPort']}" if caddy else f"http://{ip}:8000"
try:
if ip and _httpx.get(f"{url}/health", verify=False, timeout=5).status_code == 200:
yield url
return
except _httpx.TransportError:
pass
if time.time() > deadline:
pytest.fail(f"probe {name} did not serve /health within 180 s")
time.sleep(2)
@contextmanager
def legacy_key(**scope) -> Iterator[tuple[str, str]]:
"""A key written straight to the Secret the way `k7 api keys migrate` leaves a key from before roles
(``generate`` refuses such scopes); ``(name, token)``, revoked in ``finally``."""
name = f"{TENANT_PREFIX}legacy-{uuid4().hex[:6]}"
token, key_id, digest = mint_token()
now = int(time.time())
entry = {"name": name, "hash": digest, "created": now, "expires": now + 86400, **scope}
_keys_kubectl().patch(KEYS_SECRET, {key_id: json.dumps(entry)})
try:
yield name, token
finally:
_revoke_test_api_key(name)
def wait_key_seen(url: str, token: str) -> None:
"""Until ``url`` (one replica) has polled the key store and knows ``token`` (any answer but 401)."""
deadline = time.time() + 30
while call("GET", f"{url}/api/v1/auth/whoami", token, verify=False).status_code == 401:
assert time.time() < deadline, f"{url} never saw the key"
time.sleep(1)
UNSCOPED_LEGACY = {"role": "admin", "unscoped_legacy": True}
# What a migrated unscoped key would reach as admin: key minting, role bindings, remote install, sandboxes.
ADMIN_REACH = [
("POST", "/api/v1/keys", {"json": {"name": f"{TENANT_PREFIX}minted", "role": "admin", "expires_days": 1}}),
("GET", "/api/v1/keys", {}),
("GET", "/api/v1/role-bindings", {}),
("PUT", "/api/v1/role-bindings", {"json": {"bindings": [], "deny": []}}),
("POST", "/api/v1/install", {"json": {"inventory": "x"}}),
("POST", "/api/v1/tenants", {"json": {"name": f"{TENANT_PREFIX}minted", "spec": {"namespaces": ["x"]}}}),
("GET", "/api/v1/sandboxes", {"params": {"namespace": "default"}}),
("GET", "/api/v1/auth/whoami", {}),
]
class TestProductionMode:
def test_mode_is_reported(self, base):
r = _httpx.get(f"{base}/health", verify=_tls_verify(), timeout=10)
assert r.headers["X-K7-Mode"] == "development" and r.json()["mode"] == "development", r.text
@pytest.mark.parametrize(
"scope",
[UNSCOPED_LEGACY, {"role": "user", "namespaces": ["default"]}, {}],
ids=["unscoped-legacy", "ns-only", "no-role"],
)
def test_refuses_to_start_with_an_unscoped_key_and_names_it(self, base, scope):
prod = {"K7_API_MODE": "production", "K7_API_TRANSPORT": "https"}
with legacy_key(**scope) as (name, _token), _api_probe(prod) as logs:
assert "production mode refuses to start: API key(s)" in logs, logs
assert f"({name})" in logs and "--role user --tenant T" in logs, logs
def test_development_lets_an_unscoped_legacy_key_act_as_admin(self, base, admin):
with legacy_key(**UNSCOPED_LEGACY) as (name, token):
_wait_key_live(token)
me = call("GET", f"{base}/api/v1/auth/whoami", token)
assert me.status_code == 200 and me.json()["data"]["grants"] == ["admin"], me.text
for method, path, kw in ADMIN_REACH[1:3]:
r = call(method, f"{base}{path}", token, **kw)
assert r.status_code == 200, (path, r.text)
listed = call("GET", f"{base}/api/v1/keys", admin).json()["data"]
assert any(k["name"] == name and k.get("unscoped_legacy") for k in listed), listed
def test_production_refuses_an_unscoped_legacy_key_on_every_route(self, base, admin):
env = {"K7_API_MODE": "production", "K7_API_TRANSPORT": "https", "K7_API_ENABLE_REMOTE_INSTALL": "0"}
with _running_api_probe(env) as url:
wait_key_seen(url, admin)
assert call("GET", f"{url}/api/v1/auth/whoami", admin, verify=False).status_code == 200
with legacy_key(**UNSCOPED_LEGACY) as (name, token):
wait_key_seen(url, token)
for method, path, kw in ADMIN_REACH:
r = call(method, f"{url}{path}", token, verify=False, **kw)
msg = error(r)["message"]
assert r.status_code == 403 and r.headers["X-K7-Mode"] == "production", (path, r.text)
assert "production mode refuses" in msg and "unscoped_legacy" in msg, (path, msg)
assert "--role user --tenant T" in msg and f"k7 revoke-api-key {name}" in msg, (path, msg)
with legacy_key(role="user", namespaces=["default"]) as (_name, token):
wait_key_seen(url, token)
r = call("GET", f"{url}/api/v1/sandboxes", token, verify=False, params={"namespace": "default"})
assert r.status_code == 403 and "k7 tenant adopt" in error(r)["message"], r.text
listed = call("GET", f"{base}/api/v1/keys", admin).json()["data"]
assert not [k for k in listed if k["name"] == f"{TENANT_PREFIX}minted"], "a refused key minted a key"
assert kubectl("get", "tenant", f"{TENANT_PREFIX}minted").returncode != 0
def test_refuses_insecure_http(self, base):
with _api_probe({"K7_API_MODE": "production", "K7_API_TRANSPORT": "insecure-http"}) as logs:
assert "production mode refuses K7_API_TRANSPORT='insecure-http'" in logs, logs
def test_refuses_remote_install(self, base):
with _api_probe({"K7_API_MODE": "production", "K7_API_ENABLE_REMOTE_INSTALL": "1"}) as logs:
assert "production mode refuses K7_API_ENABLE_REMOTE_INSTALL=1" in logs, logs
def test_unknown_mode_refuses_to_start(self, base):
with _api_probe({"K7_API_MODE": "staging"}) as logs:
assert "K7_API_MODE='staging'; allowed: production, development" in logs, logs