Files
k7/tests/unit/test_runtime_annotations.py
G 3c80778156 Release 0.5.0
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
2026-10-18 23:21:48 +02:00

202 lines
8.6 KiB
Python

"""Only k7 may set the pod annotations containerd forwards to the k7d shim and
Kata (admission policy k7-runtime-annotations), and containerd forwards only
the ones k7 sets."""
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
import celpy
import pytest
import yaml
from celpy import adapter
from tests.unit.conftest import DEPLOY_DIR
from tests.unit.test_helm_chart import _sets, find, render
from tests.unit.test_playbook_distro_adapter import COMBOS, _render
POLICY = DEPLOY_DIR / "manifests/k7-runtime-annotations/admission-policy.yaml"
CORE = Path(__file__).resolve().parents[2] / "src/k7/core"
K7_API = "system:serviceaccount:kube-system:k7-api"
TENANT = "system:serviceaccount:tenant-a:dev"
CLUSTER_ID = {"k7d.katakate.org/cluster-id": "c1"}
KINDS = ["Pod", "Deployment", "ReplicaSet", "StatefulSet", "DaemonSet", "Job", "ReplicationController", "CronJob"]
def _policy(docs: list[dict[str, Any]] | None = None) -> dict[str, Any]:
docs = docs or [d for d in yaml.safe_load_all(POLICY.read_text()) if d]
return find(docs, "ValidatingAdmissionPolicy", "k7-runtime-annotations")
def _cel(expr: str, act: dict[str, Any]) -> Any:
env = celpy.Environment()
out = env.program(env.compile(expr)).evaluate(adapter.json_to_cel(act))
if isinstance(out, celpy.CELEvalError):
raise AssertionError(f"{expr!r}: {out}")
return out
def admit(policy: dict[str, Any], kind: str, obj: dict, old: dict | None = None, user: str = TENANT, groups=None):
"""True if the policy admits the request (a failed matchCondition skips it)."""
request = {
"operation": "UPDATE" if old is not None else "CREATE",
"kind": {"kind": kind},
"userInfo": {"username": user, **({"groups": groups} if groups else {})},
}
act: dict[str, Any] = {"request": request, "object": obj, "oldObject": old}
spec = policy["spec"]
if not all(_cel(m["expression"], act) for m in spec["matchConditions"]):
return True
variables: dict[str, Any] = {}
for v in spec["variables"]:
variables[v["name"]] = _cel(v["expression"], {**act, "variables": variables})
act["variables"] = variables
return all(_cel(v["expression"], act) for v in spec["validations"])
def workload(kind: str, annotations: dict[str, str] | None = None, **extra: str) -> dict[str, Any]:
meta: dict[str, Any] = {"labels": {"app": "x"}}
if annotations is not None:
meta["annotations"] = {**annotations, **extra}
pod = {"metadata": meta, "spec": {"containers": [{"name": "c", "image": "busybox"}]}}
if kind == "Pod":
return pod
if kind == "CronJob":
return {"metadata": {"name": "x"}, "spec": {"jobTemplate": {"spec": {"template": pod}}}}
return {"metadata": {"name": "x"}, "spec": {"template": pod}}
@pytest.mark.parametrize("kind", KINDS)
@pytest.mark.parametrize(
"annotation",
[
CLUSTER_ID,
{"k7d.katakate.org/fork-source-vm": "abc"},
{"k7d.katakate.org/tree-adopt-cluster": "abc"},
{"io.katacontainers.config_path": "/opt/kata/share/defaults/kata-containers/configuration.toml"},
{"io.katacontainers.config.runtime.disable_guest_seccomp": "true"},
],
)
def test_tenant_cannot_create_reserved_annotation(kind, annotation):
policy = _policy()
assert not admit(policy, kind, workload(kind, annotation))
assert admit(policy, kind, workload(kind, {"team": "a"}))
assert admit(policy, kind, workload(kind))
@pytest.mark.parametrize("kind", KINDS)
def test_empty_template_metadata_is_admitted(kind):
obj = workload(kind)
pod = obj if kind == "Pod" else (obj["spec"].get("jobTemplate", {}).get("spec") or obj["spec"])["template"]
del pod["metadata"]
assert admit(_policy(), kind, obj)
@pytest.mark.parametrize(
("user", "groups", "allowed"),
[
(K7_API, None, True),
("system:admin", ["system:masters", "system:authenticated"], True),
("system:kube-controller-manager", None, True),
("system:serviceaccount:kube-system:replicaset-controller", None, True),
("system:serviceaccount:kube-system:deployment-controller", None, True),
("system:serviceaccount:kube-system:default", None, False),
("system:serviceaccount:k7-system:k7-api", None, False),
("alice", ["tenant-a-admins", "system:authenticated"], False),
(TENANT, None, False),
],
)
def test_only_k7_admins_and_controllers_are_exempt(user, groups, allowed):
obj = workload("Deployment", CLUSTER_ID)
assert admit(_policy(), "Deployment", obj, user=user, groups=groups) is allowed
def test_tenant_update_may_keep_but_not_change_reserved_annotations():
policy = _policy()
k7 = {"k7d.katakate.org/docker": "true", "io.katacontainers.config.hypervisor.default_memory": "2048"}
old = workload("Deployment", k7)
assert admit(policy, "Deployment", workload("Deployment", k7, **{"kubectl.kubernetes.io/restartedAt": "now"}), old)
assert not admit(policy, "Deployment", workload("Deployment", {**k7, **CLUSTER_ID}), old)
assert not admit(policy, "Deployment", workload("Deployment", {**k7, "k7d.katakate.org/docker": "false"}), old)
assert not admit(policy, "Deployment", workload("Deployment", {"k7d.katakate.org/docker": "true"}), old)
assert admit(policy, "Deployment", workload("Deployment", {**k7, **CLUSTER_ID}), old, user=K7_API)
def test_policy_shape_and_binding():
docs = [d for d in yaml.safe_load_all(POLICY.read_text()) if d]
spec = _policy(docs)["spec"]
assert spec["failurePolicy"] == "Fail"
rules = {
(g, r)
for rule in spec["matchConstraints"]["resourceRules"]
for g in rule["apiGroups"]
for r in rule["resources"]
}
assert all(rule["operations"] == ["CREATE", "UPDATE"] for rule in spec["matchConstraints"]["resourceRules"])
assert rules == {
("", "pods"),
("", "replicationcontrollers"),
*(("apps", r) for r in ("deployments", "replicasets", "statefulsets", "daemonsets")),
("batch", "jobs"),
("batch", "cronjobs"),
}
binding = find(docs, "ValidatingAdmissionPolicyBinding", "k7-runtime-annotations")
assert binding["spec"] == {"policyName": "k7-runtime-annotations", "validationActions": ["Deny"]}
@pytest.mark.parametrize("namespace", ["kube-system", "k7-system"])
def test_chart_policy_matches_playbook(namespace):
docs = render(_sets(), namespace=namespace)
want = yaml.safe_load_all(POLICY.read_text().replace("kube-system:k7-api", f"{namespace}:k7-api"))
for doc in (d for d in want if d):
assert find(docs, doc["kind"], doc["metadata"]["name"])["spec"] == doc["spec"]
chart = _policy(docs)
assert admit(
chart, "Deployment", workload("Deployment", CLUSTER_ID), user=f"system:serviceaccount:{namespace}:k7-api"
)
def test_playbook_applies_policy_without_the_api_gate():
tasks = yaml.safe_load((DEPLOY_DIR / "roles/k7_cluster/tasks/nodes.yml").read_text())
(task,) = [t for t in tasks if "k7-runtime-annotations/admission-policy.yaml" in str(t)]
assert "when" not in task and task["run_once"] is True
def _forwarded(runtime: str) -> set[str]:
toml = _render("drop-in", (1, 1, 1, 1))
block = toml.split(f"containerd.runtimes.{runtime}]\n", 1)[1].split("\n[", 1)[0]
(line,) = re.findall(r"pod_annotations = \[(.*)\]", block)
return {s.strip(' "') for s in line.split(",")}
def test_containerd_forwards_exactly_what_core_sets():
literals = set()
for f in CORE.glob("*.py"):
literals |= set(re.findall(r'"((?:k7d\.katakate\.org/|io\.katacontainers\.)[a-z0-9_./-]+)"', f.read_text()))
k7d = {a for a in literals if a.startswith("k7d.")}
kata = literals - k7d
assert k7d == {
"k7d.katakate.org/docker",
"k7d.katakate.org/docker-disk",
"k7d.katakate.org/fork-source-cluster",
"k7d.katakate.org/fork-source-vm",
}
assert kata == {
f"io.katacontainers.config.hypervisor.{k}"
for k in ("default_memory", "rx_rate_limiter_max_rate", "tx_rate_limiter_max_rate")
}
assert _forwarded("k7") == _forwarded("k7-fc") == k7d
assert _forwarded("kata") == _forwarded("kata-qemu") == kata
for combo in COMBOS:
assert "*" not in "".join(re.findall(r"pod_annotations = .*", _render("k3s-template", combo)))
def test_kata_enables_only_the_memory_annotation():
tasks = yaml.safe_load((DEPLOY_DIR / "roles/k7_node_prep/tasks/runtimes.yml").read_text())
lines = [t["ansible.builtin.lineinfile"]["line"] for t in tasks if "enable_annotations" in str(t)]
assert (
lines == ['enable_annotations = ["default_memory", "rx_rate_limiter_max_rate", "tx_rate_limiter_max_rate"]'] * 2
)