mirror of
https://github.com/Katakate/k7.git
synced 2026-11-18 18:04:20 +00:00
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
231 lines
9.8 KiB
Python
231 lines
9.8 KiB
Python
"""k7-api side of metering: GET /api/v1/usage, requireDelivery, create-time stamps."""
|
|
|
|
import dataclasses
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
from kubernetes_asyncio import client as k
|
|
|
|
from k7.core import tenancy
|
|
from k7.core.core import _stamp_metering
|
|
from k7.metering import MEMORY_ANN, SANDBOX_UID_ANN, VCPU_ANN, config, delivery, showback
|
|
from tests.unit.auth_fixtures import api
|
|
from tests.unit.conftest import METERING
|
|
|
|
T0 = 1_767_225_600
|
|
|
|
|
|
def _cfg(**over) -> config.Config:
|
|
return dataclasses.replace(config.parse(METERING), **over)
|
|
|
|
|
|
def _prometheus(status: int = 200):
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
if status != 200:
|
|
return httpx.Response(status, text="prometheus down")
|
|
metric = req.url.params["query"].split("{")[0].split("[")[0]
|
|
result = []
|
|
if metric == "k7_usage_sandbox_seconds_total":
|
|
labels = {"tenant": "acme", "namespace": "default", "backend": "k7d", "state": "running"}
|
|
result = [{"metric": labels, "values": [[T0 + 10, "15"], [T0 + 25, "30"]]}]
|
|
return httpx.Response(200, json={"status": "success", "data": {"resultType": "matrix", "result": result}})
|
|
|
|
real = showback.usage
|
|
|
|
async def usage(url, tenant, start, end, groups, _http):
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
return await real(url, tenant, start, end, groups, http)
|
|
|
|
return patch("k7.api.main.showback.usage", usage)
|
|
|
|
|
|
@pytest.fixture()
|
|
def usage_api():
|
|
with api() as a, patch("k7.api.main.METERING", _cfg(prometheus_url="http://prom:9090")), _prometheus():
|
|
a.add_key("acme-user", role="user", tenant="acme")
|
|
a.add_key("operator", role="operator")
|
|
a.add_key("ns-operator", role="operator", namespaces=["default"])
|
|
yield a
|
|
|
|
|
|
PARAMS = {"from": str(T0), "to": "2026-01-01T00:01:00Z"}
|
|
|
|
|
|
async def test_tenant_key_sees_only_its_own_tenant(usage_api):
|
|
r = await usage_api.call("GET", "/api/v1/usage", "acme-user", params={**PARAMS, "tenant": "acme"})
|
|
assert r.status_code == 200, r.text
|
|
data = r.json()["data"]
|
|
assert data["rows"][0]["tenant"] == "acme" and data["rows"][0]["sandbox_seconds"] == 30
|
|
assert (data["from"], data["to"], data["group_by"]) == (T0, T0 + 60, ["tenant"])
|
|
for params in ({**PARAMS, "tenant": "globex"}, PARAMS):
|
|
r = await usage_api.call("GET", "/api/v1/usage", "acme-user", params=params)
|
|
assert r.status_code == 403, r.text
|
|
|
|
|
|
async def test_cluster_wide_needs_an_unscoped_operator(usage_api):
|
|
assert (await usage_api.call("GET", "/api/v1/usage", "operator", params=PARAMS)).status_code == 200
|
|
assert (await usage_api.call("GET", "/api/v1/usage", "ns-operator", params=PARAMS)).status_code == 403
|
|
assert (await usage_api.call("GET", "/api/v1/usage", None, params=PARAMS)).status_code == 401
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("params", "needle"),
|
|
[
|
|
({"from": "yesterday", "to": "60"}, "'from' must be RFC 3339"),
|
|
({"from": "2026-01-01T00:00:00", "to": "60"}, "UTC offset"),
|
|
({"from": "60", "to": "60"}, "after"),
|
|
({**PARAMS, "group_by": "sandbox"}, "group_by"),
|
|
({**PARAMS, "tenant": "Bad_Tenant"}, "valid tenant"),
|
|
],
|
|
)
|
|
async def test_bad_queries_are_400(usage_api, params, needle):
|
|
r = await usage_api.call("GET", "/api/v1/usage", "operator", params=params)
|
|
assert r.status_code == 400 and needle in r.text, r.text
|
|
|
|
|
|
async def test_unset_prometheus_url_names_the_key():
|
|
with api() as a, patch("k7.api.main.METERING", _cfg()):
|
|
a.add_key("operator", role="operator")
|
|
r = await a.call("GET", "/api/v1/usage", "operator", params=PARAMS)
|
|
assert r.status_code == 400 and "metering.prometheusUrl is unset" in r.text
|
|
|
|
|
|
async def test_prometheus_failure_is_502():
|
|
with api() as a, patch("k7.api.main.METERING", _cfg(prometheus_url="http://prom:9090")), _prometheus(503):
|
|
a.add_key("operator", role="operator")
|
|
r = await a.call("GET", "/api/v1/usage", "operator", params=PARAMS)
|
|
assert r.status_code == 502 and r.json()["error"]["code"] == "PrometheusUnavailable" and "prometheus down" in r.text
|
|
|
|
|
|
# --- requireDelivery ---------------------------------------------------------
|
|
|
|
|
|
def _dep(node_selector=None):
|
|
pod = k.V1PodSpec(containers=[k.V1Container(name="sandbox")], node_selector=node_selector)
|
|
return k.V1Deployment(spec=k.V1DeploymentSpec(selector=k.V1LabelSelector(), template=k.V1PodTemplateSpec(spec=pod)))
|
|
|
|
|
|
def _status(over: bool) -> dict:
|
|
return {"spool": {"above_high_water": over, "bytes": 900, "max_bytes": 1000, "high_water_percent": 80}}
|
|
|
|
|
|
def _core(spools: dict[str, bool | None]):
|
|
async def agent(node, method, path, timeout):
|
|
assert (method, path) == ("GET", "/agent/v1/usage/status")
|
|
if spools[node] is None:
|
|
raise RuntimeError("agent unreachable")
|
|
return _status(bool(spools[node]))
|
|
|
|
v1 = AsyncMock()
|
|
v1.list_node.return_value = SimpleNamespace(
|
|
items=[SimpleNamespace(metadata=SimpleNamespace(name=n)) for n in spools]
|
|
)
|
|
return SimpleNamespace(_agent_request=agent, _get_core_v1_client=AsyncMock(return_value=v1))
|
|
|
|
|
|
@pytest.fixture()
|
|
def require_delivery(metering_env):
|
|
metering_env.write_text(json.dumps({**METERING, "requireDelivery": True}))
|
|
|
|
|
|
async def test_off_by_default_checks_nothing():
|
|
core = SimpleNamespace(_agent_request=AsyncMock(side_effect=AssertionError("called")))
|
|
await delivery.require(core, _dep(), None)
|
|
|
|
|
|
async def test_pinned_create_on_an_overflowing_node_is_503(require_delivery):
|
|
with pytest.raises(tenancy.AdmissionError) as e:
|
|
await delivery.require(_core({"n1": True, "n2": False}), _dep({delivery.HOSTNAME: "n1"}), None)
|
|
assert (e.value.status, e.value.code) == (503, "UsageSpoolFull")
|
|
assert "node n1" in e.value.message and "above its 80% high-water mark" in e.value.message
|
|
|
|
|
|
async def test_unpinned_create_is_steered_off_overflowing_and_unknown_nodes(require_delivery):
|
|
dep = _dep()
|
|
await delivery.require(_core({"n1": True, "n2": False, "n3": None}), dep, None)
|
|
(term,) = (
|
|
dep.spec.template.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms
|
|
)
|
|
(expr,) = term.match_expressions
|
|
assert (expr.key, expr.operator, expr.values) == (delivery.HOSTNAME, "NotIn", ["n1", "n3"])
|
|
|
|
|
|
async def test_tenant_nodes_bound_the_candidates_and_all_over_is_503(require_delivery):
|
|
tenant = {"spec": {"nodes": ["n1"]}}
|
|
with pytest.raises(tenancy.AdmissionError, match="usage spool state unknown"):
|
|
await delivery.require(_core({"n1": None, "n2": False}), _dep(), tenant)
|
|
|
|
|
|
async def test_existing_affinity_terms_gain_the_exclusion(require_delivery):
|
|
dep = _dep()
|
|
req = k.V1NodeSelectorRequirement(key="k7.katakate.org/backend-k7d", operator="Exists")
|
|
dep.spec.template.spec.affinity = k.V1Affinity(
|
|
node_affinity=k.V1NodeAffinity(
|
|
required_during_scheduling_ignored_during_execution=k.V1NodeSelector(
|
|
node_selector_terms=[k.V1NodeSelectorTerm(match_expressions=[req])]
|
|
)
|
|
)
|
|
)
|
|
await delivery.require(_core({"n1": True, "n2": False}), dep, None)
|
|
(term,) = (
|
|
dep.spec.template.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms
|
|
)
|
|
assert [e.operator for e in term.match_expressions] == ["Exists", "NotIn"]
|
|
|
|
|
|
# --- create-time stamps ------------------------------------------------------
|
|
|
|
|
|
def test_stamp_metering_sets_a_fresh_uid_and_the_reserved_size():
|
|
res = k.V1ResourceRequirements(limits={"cpu": "1500m", "memory": "2Gi"})
|
|
tmpl = k.V1PodTemplateSpec(
|
|
metadata=k.V1ObjectMeta(annotations={VCPU_ANN: "9", SANDBOX_UID_ANN: "copied-from-source"}),
|
|
spec=k.V1PodSpec(containers=[k.V1Container(name="sidecar"), k.V1Container(name="sandbox", resources=res)]),
|
|
)
|
|
dep = k.V1Deployment(
|
|
metadata=k.V1ObjectMeta(name="sb"), spec=k.V1DeploymentSpec(selector=k.V1LabelSelector(), template=tmpl)
|
|
)
|
|
_stamp_metering(dep)
|
|
for meta in (dep.metadata, dep.spec.template.metadata):
|
|
assert meta.annotations[VCPU_ANN] == "1.5" and meta.annotations[MEMORY_ANN] == str(2 * 2**30)
|
|
assert meta.annotations[SANDBOX_UID_ANN] != "copied-from-source"
|
|
assert dep.metadata.annotations[SANDBOX_UID_ANN] == dep.spec.template.metadata.annotations[SANDBOX_UID_ANN]
|
|
unsized = k.V1Deployment(
|
|
metadata=k.V1ObjectMeta(annotations={MEMORY_ANN: "1"}),
|
|
spec=k.V1DeploymentSpec(
|
|
selector=k.V1LabelSelector(),
|
|
template=k.V1PodTemplateSpec(
|
|
metadata=k.V1ObjectMeta(), spec=k.V1PodSpec(containers=[k.V1Container(name="sandbox")])
|
|
),
|
|
),
|
|
)
|
|
_stamp_metering(unsized)
|
|
assert set(unsized.metadata.annotations) == {SANDBOX_UID_ANN}
|
|
|
|
|
|
async def test_every_create_path_is_stamped_and_gated(require_delivery):
|
|
from k7.core.core import K7Core
|
|
|
|
core = K7Core()
|
|
apps = AsyncMock()
|
|
core._get_apps_v1_client = AsyncMock(return_value=apps)
|
|
dep = k.V1Deployment(
|
|
metadata=k.V1ObjectMeta(name="sb"),
|
|
spec=k.V1DeploymentSpec(
|
|
selector=k.V1LabelSelector(),
|
|
template=k.V1PodTemplateSpec(
|
|
metadata=k.V1ObjectMeta(), spec=k.V1PodSpec(containers=[k.V1Container(name="sandbox")])
|
|
),
|
|
),
|
|
)
|
|
await core._create_sandbox_deployment("default", dep)
|
|
assert SANDBOX_UID_ANN in dep.metadata.annotations and apps.create_namespaced_deployment.await_count == 1
|
|
core.admission = SimpleNamespace(tenant=None)
|
|
full = AsyncMock(side_effect=tenancy.AdmissionError(503, "UsageSpoolFull", "full", {}))
|
|
with patch("k7.core.core.delivery.require", full), pytest.raises(tenancy.AdmissionError):
|
|
await core._create_sandbox_deployment("default", dep)
|
|
assert apps.create_namespaced_deployment.await_count == 1
|