mirror of
https://github.com/Katakate/k7.git
synced 2026-11-18 18:04:20 +00:00
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
177 lines
7.0 KiB
Python
177 lines
7.0 KiB
Python
"""k7-agent metering: spool ingest, storage hand-off, health and the usage endpoints."""
|
|
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from k7 import metrics
|
|
from k7.metering import config
|
|
from k7.metering.events import DISK, GAP, SANDBOX, envelope, event_id
|
|
from k7.metering.node import NodeMetering
|
|
from k7.metering.sampler import parse_pods
|
|
from tests.unit.conftest import CLUSTER_ID
|
|
from tests.unit.test_agent import TOKEN, _client
|
|
from tests.unit.test_metering_events import sandbox_event
|
|
from tests.unit.test_metering_sampler import _item
|
|
|
|
H = {"X-K7-Agent-Token": TOKEN}
|
|
STORAGE = f"k7://{CLUSTER_ID}/storage"
|
|
|
|
|
|
def disk_event(source: str = STORAGE) -> dict:
|
|
data = {
|
|
"tenant": "acme",
|
|
"namespace": "acme-dev",
|
|
"sandbox": "sb",
|
|
"sandbox_uid": "u-1",
|
|
"api_key_id": None,
|
|
"backend": "kata-qemu-longhorn",
|
|
"state": "stopped",
|
|
"window_start": "2026-01-01T00:00:00.000Z",
|
|
"window_end": "2026-01-01T00:01:00.000Z",
|
|
"seconds": 60.0,
|
|
"bytes": 2**30,
|
|
"gib_seconds": 60.0,
|
|
}
|
|
eid = event_id(CLUSTER_ID, "u-1", "disk", data["window_start"])
|
|
return envelope(DISK, source, "acme/acme-dev/sb", eid, data["window_end"], data)
|
|
|
|
|
|
def node(tmp_path) -> NodeMetering:
|
|
return NodeMetering(config.load(), CLUSTER_ID, "node-a", tmp_path)
|
|
|
|
|
|
def _all(m: NodeMetering) -> list[dict]:
|
|
m.spool.cursors["probe"] = (m.spool._seqs()[0], 0)
|
|
return m.spool.read("probe", 1000)[0]
|
|
|
|
|
|
def test_ingest_validates_before_spooling(tmp_path):
|
|
m = node(tmp_path)
|
|
with pytest.raises(ValueError, match="state"):
|
|
m.ingest([sandbox_event(), sandbox_event(state="stopped")])
|
|
assert m.spool.size() == 0
|
|
|
|
|
|
def test_storage_hand_off_accepts_only_storage_events(tmp_path):
|
|
m = node(tmp_path)
|
|
m.ingest_storage([disk_event()])
|
|
for wrong in (sandbox_event(), disk_event(source=f"k7://{CLUSTER_ID}/node/node-a")):
|
|
with pytest.raises(ValueError, match="only"):
|
|
m.ingest_storage([wrong])
|
|
assert [e["type"] for e in _all(m)] == [DISK]
|
|
|
|
|
|
async def test_sample_once_spools_gap_then_windows(tmp_path):
|
|
m = node(tmp_path)
|
|
crictl = {"pods": {"items": [_item()]}, "statsp": {"stats": []}}
|
|
with patch("k7.metering.node.crictl", side_effect=lambda *a: crictl[a[0]]):
|
|
await m.sample_once()
|
|
await m.sample_once()
|
|
assert [e["type"] for e in _all(m)] == [GAP, SANDBOX]
|
|
assert m.last_sample is not None and m.sampler.egress_unavailable == {"k7d": True}
|
|
assert metrics.AGENT.get_sample_value("k7_usage_egress_unavailable", {"backend": "k7d"}) == 1
|
|
assert (tmp_path / "sampler-state.json").exists()
|
|
|
|
|
|
async def test_cancel_during_spool_append_still_saves_the_sampler_state(tmp_path):
|
|
"""A shutdown mid-sample must not leave spooled windows behind a stale state file."""
|
|
m = node(tmp_path)
|
|
crictl = {"pods": {"items": [_item()]}, "statsp": {"stats": []}}
|
|
entered, release = threading.Event(), threading.Event()
|
|
ingest = m.ingest
|
|
|
|
def slow_ingest(events):
|
|
entered.set()
|
|
release.wait(5)
|
|
ingest(events)
|
|
|
|
with patch("k7.metering.node.crictl", side_effect=lambda *a: crictl[a[0]]):
|
|
await m.sample_once()
|
|
m.ingest = slow_ingest
|
|
task = asyncio.create_task(m.sample_once())
|
|
await asyncio.to_thread(entered.wait, 5)
|
|
task.cancel()
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
await asyncio.sleep(0.2)
|
|
saved = json.loads((tmp_path / "sampler-state.json").read_text())["t_wall"]
|
|
windows = [e for e in _all(m) if e["type"] == SANDBOX]
|
|
assert len(windows) == 1
|
|
assert abs(saved - datetime.fromisoformat(windows[-1]["data"]["window_end"][:-1] + "+00:00").timestamp()) < 0.002
|
|
|
|
|
|
async def test_statsp_failure_is_egress_unavailable_not_a_failed_sample(tmp_path):
|
|
m = node(tmp_path)
|
|
|
|
def crictl(*args):
|
|
if args[0] == "statsp":
|
|
raise RuntimeError("statsp: not implemented")
|
|
return {"items": [_item()]}
|
|
|
|
with patch("k7.metering.node.crictl", side_effect=crictl):
|
|
await m.sample_once()
|
|
assert m.sampler.egress_unavailable == {"k7d": True}
|
|
|
|
|
|
def test_gap_events_are_counted(tmp_path):
|
|
m = node(tmp_path)
|
|
labels = {"meter": "sandbox", "reason": "metering-start"}
|
|
before = metrics.AGENT.get_sample_value("k7_usage_gaps_total", labels) or 0
|
|
pods = parse_pods({"items": [_item()]})[0]
|
|
m.ingest(m.sampler.step(pods, {"cri-1": None}, {}, 1_767_225_600.0, 0.0))
|
|
assert metrics.AGENT.get_sample_value("k7_usage_gaps_total", labels) == before + 1
|
|
|
|
|
|
def test_health_turns_degraded_with_reasons(tmp_path):
|
|
m = node(tmp_path)
|
|
assert m.health()["status"] == "healthy"
|
|
m.sample_error = "crictl failed"
|
|
m.unmetered = ["ns/pod: unknown backend"]
|
|
m.spool.last_drop = time.time()
|
|
m.sampler.egress_unavailable = {"k7d": True, "kata-qemu-longhorn": False}
|
|
h = m.health()
|
|
assert h["status"] == "degraded" and h["egress_unavailable"] == ["k7d"]
|
|
assert len(h["degraded"]) == 3 and any("spool overflow" in r for r in h["degraded"])
|
|
|
|
|
|
async def test_usage_endpoints(tmp_path, monkeypatch):
|
|
transport = _client(tmp_path, monkeypatch)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://agent") as c:
|
|
assert (await c.post("/agent/v1/usage/events", json={"events": [disk_event()]})).status_code == 401
|
|
r = await c.post("/agent/v1/usage/events", json={"events": [disk_event()]}, headers=H)
|
|
assert r.status_code == 200 and r.json() == {"accepted": 1}
|
|
r = await c.post("/agent/v1/usage/events", json={"events": [sandbox_event()]}, headers=H)
|
|
assert r.status_code == 400 and "only" in r.json()["detail"]
|
|
assert (await c.post("/agent/v1/usage/events", json={"events": []}, headers=H)).status_code == 400
|
|
status = (await c.get("/agent/v1/usage/status", headers=H)).json()
|
|
assert status["spool"]["bytes"] > 0 and status["spool"]["above_high_water"] is False
|
|
health = (await c.get("/health")).json()
|
|
assert health["status"] == "healthy" and health["usage"]["sample_interval"] == 15
|
|
|
|
|
|
async def test_pause_and_resume_record_transitions(tmp_path, monkeypatch):
|
|
from unittest.mock import AsyncMock
|
|
|
|
from k7.metering import pauses
|
|
|
|
transport = _client(tmp_path, monkeypatch)
|
|
vm = {"vm_id": "vm-1", "status": "sandbox_found", "sandbox_id": "cri-9"}
|
|
body = {"name": "sb", "namespace": "ns", "pod_name": "sb-pod"}
|
|
with (
|
|
patch("k7.api.agent._lookup_local_vm", new=AsyncMock(return_value=vm)),
|
|
patch("k7.api.agent.K7Core") as core_cls,
|
|
):
|
|
core_cls.return_value._k7d_request = AsyncMock(return_value={"status": "ok"})
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://agent") as c:
|
|
assert (await c.post("/agent/v1/vm/pause", json=body, headers=H)).status_code == 200
|
|
assert (await c.post("/agent/v1/vm/resume", json=body, headers=H)).status_code == 200
|
|
assert [p for _, p in pauses.load()["cri-9"]] == [True, False]
|