mirror of
https://github.com/Katakate/k7.git
synced 2026-11-18 18:04:20 +00:00
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
1103 lines
39 KiB
Python
1103 lines
39 KiB
Python
"""Integration tests for the K7 API running as a K3s pod.
|
|
|
|
These tests exercise the full HTTP path: request → NodePort → k7-api pod
|
|
→ FastAPI → K7Core → Kubernetes API. They verify that the API deployment,
|
|
ServiceAccount RBAC, and in-cluster config all work end-to-end.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx as _httpx
|
|
import pytest
|
|
|
|
from .cluster import API_CA, KUBECTL, system_ns
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
class _Httpx:
|
|
"""httpx wrapper that trusts the playbook CA when the API is HTTPS."""
|
|
|
|
ConnectError = _httpx.ConnectError
|
|
|
|
def get(self, *args, **kwargs):
|
|
kwargs.setdefault("verify", _tls_verify())
|
|
return _httpx.get(*args, **kwargs)
|
|
|
|
def post(self, *args, **kwargs):
|
|
kwargs.setdefault("verify", _tls_verify())
|
|
return _httpx.post(*args, **kwargs)
|
|
|
|
def delete(self, *args, **kwargs):
|
|
kwargs.setdefault("verify", _tls_verify())
|
|
return _httpx.delete(*args, **kwargs)
|
|
|
|
def Client(self, *args, **kwargs):
|
|
kwargs.setdefault("verify", _tls_verify())
|
|
return _httpx.Client(*args, **kwargs)
|
|
|
|
|
|
httpx = _Httpx()
|
|
|
|
_API_ENDPOINT_FILE = Path("/etc/k7/api_endpoint")
|
|
|
|
|
|
def _tls_verify() -> bool | str:
|
|
"""httpx/SDK ``verify`` for the current install."""
|
|
endpoint = _API_ENDPOINT_FILE.read_text().strip() if _API_ENDPOINT_FILE.exists() else ""
|
|
if endpoint.startswith("http://"):
|
|
return False
|
|
if API_CA.exists():
|
|
return str(API_CA)
|
|
return bool(endpoint.startswith("https://"))
|
|
|
|
|
|
def _api_scheme() -> str:
|
|
verify = _tls_verify()
|
|
return "http" if verify is False else "https"
|
|
|
|
|
|
def _get_api_base_url() -> str:
|
|
"""Read the API endpoint from the K3s service."""
|
|
result = subprocess.run(
|
|
[*KUBECTL, "get", "svc", "k7-api", "-n", system_ns(), "-o", "jsonpath={.spec.ports[0].nodePort}"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode != 0 or not result.stdout.strip():
|
|
pytest.skip("k7-api service not found — run 'k7 install' or 'k7 api enable' first")
|
|
port = result.stdout.strip()
|
|
return f"{_api_scheme()}://127.0.0.1:{port}"
|
|
|
|
|
|
def _api_pod_ready() -> bool:
|
|
result = subprocess.run(
|
|
[
|
|
*KUBECTL,
|
|
"get",
|
|
"deployment",
|
|
"k7-api",
|
|
"-n",
|
|
system_ns(),
|
|
"-o",
|
|
"jsonpath={.status.readyReplicas}",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return result.returncode == 0 and result.stdout.strip() not in ("", "0")
|
|
|
|
|
|
def _keys_kubectl():
|
|
from k7.deploy.k7_api_keys import Kubectl
|
|
|
|
return Kubectl(KUBECTL, system_ns())
|
|
|
|
|
|
def _wait_key_live(token: str) -> None:
|
|
"""Keys reach every API replica within one store poll (5 s)."""
|
|
from k7.api.keystore import POLL_SECONDS
|
|
|
|
url = _get_api_base_url()
|
|
deadline = time.monotonic() + 4 * POLL_SECONDS
|
|
while time.monotonic() < deadline:
|
|
if httpx.get(f"{url}/api/v1/sandboxes/metrics", headers={"X-API-Key": token}, timeout=5).status_code != 401:
|
|
time.sleep(POLL_SECONDS + 1) # the other replica's poll
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"API key not accepted within {4 * POLL_SECONDS}s of writing the k7-api-keys Secret")
|
|
|
|
|
|
def _generate_test_api_key(
|
|
*,
|
|
name: str,
|
|
namespaces: list[str] | None = None,
|
|
nodes: list[str] | None = None,
|
|
role: str = "admin",
|
|
tenant: str | None = None,
|
|
) -> str:
|
|
"""Create a 1-day key in the k7-api-keys Secret, wait until the API accepts it, return the token."""
|
|
from k7.deploy.k7_api_keys import generate
|
|
|
|
token, _key_id, _entry = generate(
|
|
_keys_kubectl(),
|
|
name,
|
|
role=role,
|
|
tenant=tenant,
|
|
expires_days=1,
|
|
namespaces=list(namespaces or []),
|
|
nodes=list(nodes or []),
|
|
)
|
|
_wait_key_live(token)
|
|
return token
|
|
|
|
|
|
def _revoke_test_api_key(name: str) -> None:
|
|
"""Remove test keys by name from the Secret (try/finally cleanup)."""
|
|
from k7.deploy.k7_api_keys import revoke
|
|
|
|
revoke(_keys_kubectl(), name)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def api_base_url() -> str:
|
|
if not _api_pod_ready():
|
|
pytest.skip("k7-api deployment not ready")
|
|
url = _get_api_base_url()
|
|
client = httpx.Client()
|
|
for _ in range(30):
|
|
try:
|
|
r = client.get(f"{url}/health", timeout=3)
|
|
if r.status_code == 200:
|
|
client.close()
|
|
return url
|
|
except httpx.ConnectError:
|
|
pass
|
|
time.sleep(2)
|
|
client.close()
|
|
pytest.skip("k7-api health endpoint not reachable")
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def api_key() -> Iterator[str]:
|
|
name = f"integ-api-{uuid4().hex[:8]}"
|
|
try:
|
|
yield _generate_test_api_key(name=name)
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def api_headers(api_key: str) -> dict[str, str]:
|
|
return {"X-API-Key": api_key}
|
|
|
|
|
|
class TestApiHealth:
|
|
def test_root_returns_version(self, api_base_url: str):
|
|
r = httpx.get(f"{api_base_url}/", timeout=5)
|
|
assert r.status_code == 200
|
|
data = r.json()
|
|
assert "version" in data
|
|
assert data["message"] == "K7 Sandbox API"
|
|
|
|
def test_health_returns_healthy(self, api_base_url: str):
|
|
r = httpx.get(f"{api_base_url}/health", timeout=5)
|
|
assert r.status_code == 200
|
|
assert r.json()["status"] == "healthy"
|
|
|
|
|
|
class TestApiAuth:
|
|
def test_list_without_key_returns_401(self, api_base_url: str):
|
|
r = httpx.get(f"{api_base_url}/api/v1/sandboxes", timeout=5)
|
|
assert r.status_code == 401
|
|
|
|
def test_list_with_bad_key_returns_401(self, api_base_url: str):
|
|
r = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
headers={"X-API-Key": "bogus-key"},
|
|
timeout=5,
|
|
)
|
|
assert r.status_code == 401
|
|
|
|
def test_list_with_valid_key_returns_200(self, api_base_url: str, api_headers: dict):
|
|
r = httpx.get(f"{api_base_url}/api/v1/sandboxes", headers=api_headers, timeout=5)
|
|
assert r.status_code == 200
|
|
assert "data" in r.json()
|
|
|
|
|
|
class TestApiSandboxLifecycle:
|
|
"""Create → list → get → exec → delete, all through the HTTP API."""
|
|
|
|
SANDBOX_NAME = "api-integ-test"
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _cleanup(self, api_base_url: str, api_headers: dict, test_namespace: str):
|
|
yield
|
|
httpx.delete(
|
|
f"{api_base_url}/api/v1/sandboxes/{self.SANDBOX_NAME}",
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=10,
|
|
)
|
|
|
|
def test_create_list_exec_delete(self, api_base_url: str, api_headers: dict, test_namespace: str):
|
|
# Create
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={"name": self.SANDBOX_NAME, "image": "alpine:3.20", "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert r.status_code == 201, f"create failed: {r.text}"
|
|
assert r.json()["data"]["name"] == self.SANDBOX_NAME
|
|
|
|
# Wait for ready
|
|
ready = False
|
|
for _ in range(90):
|
|
gr = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes/{self.SANDBOX_NAME}",
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=5,
|
|
)
|
|
if gr.status_code == 200 and gr.json()["data"].get("ready") == "True":
|
|
ready = True
|
|
break
|
|
time.sleep(2)
|
|
assert ready, "sandbox never became ready"
|
|
|
|
# List should include it
|
|
lr = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=5,
|
|
)
|
|
assert lr.status_code == 200
|
|
names = [s["name"] for s in lr.json()["data"]]
|
|
assert self.SANDBOX_NAME in names
|
|
|
|
# Exec
|
|
er = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{self.SANDBOX_NAME}/exec",
|
|
json={"command": "echo api-works"},
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert er.status_code == 200
|
|
exec_data = er.json()["data"]
|
|
assert exec_data["exit_code"] == 0
|
|
assert "api-works" in exec_data["stdout"]
|
|
|
|
# Delete
|
|
dr = httpx.delete(
|
|
f"{api_base_url}/api/v1/sandboxes/{self.SANDBOX_NAME}",
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert dr.status_code == 200, dr.text
|
|
|
|
# Confirm gone
|
|
gr2 = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes/{self.SANDBOX_NAME}",
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=5,
|
|
)
|
|
assert gr2.status_code == 404
|
|
|
|
|
|
def _wait_for_sandbox_ready(api_base_url: str, api_headers: dict, name: str, namespace: str, timeout: int = 240):
|
|
"""Poll the API until the sandbox reports Ready=True. Raises on timeout."""
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
r = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}",
|
|
params={"namespace": namespace},
|
|
headers=api_headers,
|
|
timeout=5,
|
|
)
|
|
if r.status_code == 200 and r.json()["data"].get("ready") == "True":
|
|
return
|
|
time.sleep(2)
|
|
raise TimeoutError(f"Sandbox {name} did not become ready within {timeout}s")
|
|
|
|
|
|
def _replicas(name: str, namespace: str) -> int:
|
|
out = subprocess.run(
|
|
[
|
|
*KUBECTL,
|
|
"get",
|
|
"deployment",
|
|
name,
|
|
"-n",
|
|
namespace,
|
|
"-o",
|
|
"jsonpath={.spec.replicas}",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if out.returncode != 0 or not out.stdout.strip():
|
|
return -1
|
|
return int(out.stdout.strip())
|
|
|
|
|
|
def _snapshot_exists(name: str, namespace: str) -> bool:
|
|
out = subprocess.run(
|
|
[
|
|
*KUBECTL,
|
|
"get",
|
|
"volumesnapshot",
|
|
name,
|
|
"-n",
|
|
namespace,
|
|
"-o",
|
|
"name",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return out.returncode == 0 and out.stdout.strip().startswith("volumesnapshot.snapshot.storage.k8s.io/")
|
|
|
|
|
|
@pytest.fixture()
|
|
def cleanup_sandbox(api_base_url: str, api_headers: dict):
|
|
"""Yield a callable that registers sandbox (name, namespace) pairs for teardown."""
|
|
to_delete: list[tuple[str, str]] = []
|
|
|
|
def _register(name: str, namespace: str) -> None:
|
|
to_delete.append((name, namespace))
|
|
|
|
yield _register
|
|
|
|
for name, namespace in to_delete:
|
|
try:
|
|
httpx.delete(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}",
|
|
params={"namespace": namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
class TestApiPauseResumeFork:
|
|
"""Pause/resume/fork via the HTTP API, multi-node cluster.
|
|
|
|
Uses kata-qemu-longhorn (only backend that supports snapshot + fork).
|
|
"""
|
|
|
|
def test_pause_resume_round_trip(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
name = "api-pr"
|
|
cleanup_sandbox(name, test_namespace)
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": name,
|
|
"image": "alpine:3.20",
|
|
"namespace": test_namespace,
|
|
"backend": "kata-qemu-longhorn",
|
|
},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert r.status_code == 201, f"create failed: {r.text}"
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, name, test_namespace)
|
|
|
|
pr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}/pause",
|
|
json={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert pr.status_code == 200, pr.text
|
|
# Wait for the scale change to settle.
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline:
|
|
if _replicas(name, test_namespace) == 0:
|
|
break
|
|
time.sleep(1)
|
|
assert _replicas(name, test_namespace) == 0, "deployment did not scale to 0 after pause"
|
|
|
|
rr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}/resume",
|
|
json={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert rr.status_code == 200, rr.text
|
|
assert _replicas(name, test_namespace) == 1, "deployment did not scale back to 1 after resume"
|
|
|
|
def test_pause_with_snapshot(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
name = "api-pr-snap"
|
|
snap = f"{name}-v1"
|
|
cleanup_sandbox(name, test_namespace)
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": name,
|
|
"image": "alpine:3.20",
|
|
"namespace": test_namespace,
|
|
"backend": "kata-qemu-longhorn",
|
|
},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert r.status_code == 201, r.text
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, name, test_namespace)
|
|
|
|
# ``snapshot`` alone is enough — pvc/snapshot_class derived server-side.
|
|
pr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}/pause",
|
|
json={"namespace": test_namespace, "snapshot": snap},
|
|
headers=api_headers,
|
|
timeout=120,
|
|
)
|
|
assert pr.status_code == 200, pr.text
|
|
|
|
deadline = time.time() + 90
|
|
snap_seen = False
|
|
while time.time() < deadline:
|
|
if _snapshot_exists(snap, test_namespace):
|
|
snap_seen = True
|
|
break
|
|
time.sleep(2)
|
|
assert snap_seen, f"VolumeSnapshot {snap} never appeared in {test_namespace}"
|
|
|
|
try:
|
|
rr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}/resume",
|
|
json={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert rr.status_code == 200, rr.text
|
|
assert _snapshot_exists(snap, test_namespace), "snapshot should survive resume"
|
|
finally:
|
|
subprocess.run(
|
|
[
|
|
*KUBECTL,
|
|
"delete",
|
|
"volumesnapshot",
|
|
snap,
|
|
"-n",
|
|
test_namespace,
|
|
"--ignore-not-found",
|
|
"--wait=false",
|
|
],
|
|
capture_output=True,
|
|
)
|
|
|
|
def test_fork_basic_via_api(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
source = "api-fork-src"
|
|
child = "api-fork-child"
|
|
cleanup_sandbox(source, test_namespace)
|
|
cleanup_sandbox(child, test_namespace)
|
|
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": source,
|
|
"image": "alpine:3.20",
|
|
"namespace": test_namespace,
|
|
"backend": "kata-qemu-longhorn",
|
|
},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert r.status_code == 201, r.text
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, source, test_namespace)
|
|
|
|
# Write a marker into the source's persistent state.
|
|
er = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{source}/exec",
|
|
json={"command": "echo fork-marker > /mnt/state/marker"},
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert er.status_code == 200, er.text
|
|
|
|
fr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{source}/fork",
|
|
json={"new_name": child, "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=600,
|
|
)
|
|
assert fr.status_code == 201, fr.text
|
|
body = fr.json()["data"]
|
|
assert body["name"] == child
|
|
assert body["source"] == source
|
|
assert fr.headers.get("location", "").endswith(f"/api/v1/sandboxes/{child}?namespace={test_namespace}")
|
|
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, child, test_namespace)
|
|
|
|
# Marker survives the fork.
|
|
cr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{child}/exec",
|
|
json={"command": "cat /mnt/state/marker"},
|
|
params={"namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert cr.status_code == 200
|
|
assert "fork-marker" in cr.json()["data"]["stdout"]
|
|
|
|
def test_fork_into_existing_name_returns_409(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
a = "api-fork-conflict-a"
|
|
b = "api-fork-conflict-b"
|
|
cleanup_sandbox(a, test_namespace)
|
|
cleanup_sandbox(b, test_namespace)
|
|
|
|
for n in (a, b):
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": n,
|
|
"image": "alpine:3.20",
|
|
"namespace": test_namespace,
|
|
"backend": "kata-qemu-longhorn",
|
|
},
|
|
headers=api_headers,
|
|
timeout=60,
|
|
)
|
|
assert r.status_code == 201, r.text
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, n, test_namespace)
|
|
|
|
fr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/{a}/fork",
|
|
json={"new_name": b, "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=600,
|
|
)
|
|
assert fr.status_code == 409, fr.text
|
|
|
|
def test_fork_missing_source_returns_404(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
):
|
|
fr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/does-not-exist/fork",
|
|
json={"new_name": "child", "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert fr.status_code == 404, fr.text
|
|
|
|
@pytest.mark.parametrize("op, body", [("pause", {}), ("snapshot", {"snapshot_name": "snap"})])
|
|
def test_pause_and_snapshot_missing_sandbox_return_404(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
op: str,
|
|
body: dict,
|
|
):
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/does-not-exist/{op}",
|
|
json={**body, "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert r.status_code == 404, r.text
|
|
|
|
def test_fork_missing_new_name_returns_400(
|
|
self,
|
|
api_base_url: str,
|
|
api_headers: dict,
|
|
test_namespace: str,
|
|
):
|
|
fr = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes/whatever/fork",
|
|
json={},
|
|
headers=api_headers,
|
|
timeout=10,
|
|
)
|
|
assert fr.status_code == 400, fr.text
|
|
|
|
|
|
class TestSdkSandboxProxy:
|
|
"""End-to-end SDK exercise: Client.create → proxy.pause/resume/fork."""
|
|
|
|
def test_sdk_pause_resume_fork_round_trip(
|
|
self,
|
|
api_base_url: str,
|
|
api_key: str,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
from k7_sdk import Client
|
|
|
|
c = Client(endpoint=api_base_url, api_key=api_key, verify_ssl=_tls_verify())
|
|
name = "sdk-prf"
|
|
child = "sdk-prf-child"
|
|
cleanup_sandbox(name, test_namespace)
|
|
cleanup_sandbox(child, test_namespace)
|
|
|
|
proxy = c.create(
|
|
{
|
|
"name": name,
|
|
"image": "alpine:3.20",
|
|
"namespace": test_namespace,
|
|
"backend": "kata-qemu-longhorn",
|
|
}
|
|
)
|
|
assert proxy.name == name
|
|
_wait_for_sandbox_ready(api_base_url, {"X-API-Key": api_key}, name, test_namespace)
|
|
|
|
proxy.pause()
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline and _replicas(name, test_namespace) != 0:
|
|
time.sleep(1)
|
|
assert _replicas(name, test_namespace) == 0
|
|
|
|
proxy.resume()
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline and _replicas(name, test_namespace) != 1:
|
|
time.sleep(1)
|
|
assert _replicas(name, test_namespace) == 1
|
|
|
|
# Need the pod Running again before exec.
|
|
_wait_for_sandbox_ready(api_base_url, {"X-API-Key": api_key}, name, test_namespace)
|
|
proxy.exec("echo sdk-fork-marker > /mnt/state/marker")
|
|
|
|
forked = proxy.fork(child)
|
|
assert forked.name == child
|
|
assert forked.namespace == test_namespace
|
|
_wait_for_sandbox_ready(api_base_url, {"X-API-Key": api_key}, child, test_namespace)
|
|
|
|
out = forked.exec("cat /mnt/state/marker")
|
|
assert "sdk-fork-marker" in out["stdout"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SSRF guard + API-key namespace authorization
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestApiSsrfGuard:
|
|
"""Control-plane must not fetch OCI manifests from internal/loopback hosts."""
|
|
|
|
def test_loopback_image_rejected_and_listener_untouched(
|
|
self, api_base_url: str, api_headers: dict, test_namespace: str
|
|
):
|
|
port = 8199
|
|
# Tiny python listener with a hit counter. Bound on all interfaces so a
|
|
# buggy control-plane fetch via the node IP would also be visible; the
|
|
# image uses 127.0.0.1 which the guard must reject before any dial.
|
|
counter = Path("/tmp/k7-ssrf-listener-hits")
|
|
if counter.exists():
|
|
counter.unlink()
|
|
listener = subprocess.Popen(
|
|
[
|
|
"python3",
|
|
"-c",
|
|
(
|
|
"import socket, pathlib\n"
|
|
f"p=pathlib.Path({str(counter)!r})\n"
|
|
"s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n"
|
|
f"s.bind(('0.0.0.0', {port})); s.listen(5); s.settimeout(8)\n"
|
|
"try:\n"
|
|
" c,_=s.accept(); p.write_text('hit'); c.close()\n"
|
|
"except Exception:\n"
|
|
" pass\n"
|
|
"finally:\n"
|
|
" s.close()\n"
|
|
),
|
|
],
|
|
)
|
|
try:
|
|
time.sleep(0.4)
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": "ssrf-loopback",
|
|
"image": f"127.0.0.1:{port}/x:1",
|
|
"namespace": test_namespace,
|
|
},
|
|
headers=api_headers,
|
|
timeout=15,
|
|
)
|
|
assert r.status_code >= 400, f"expected error, got {r.status_code}: {r.text}"
|
|
assert "not allowed" in r.text.lower() or "registry" in r.text.lower()
|
|
time.sleep(0.5)
|
|
assert not counter.exists(), "listener received a connection — SSRF guard failed"
|
|
finally:
|
|
listener.kill()
|
|
try:
|
|
listener.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
|
|
def test_metadata_image_rejected(self, api_base_url: str, api_headers: dict, test_namespace: str):
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": "ssrf-metadata",
|
|
"image": "169.254.169.254/latest/meta-data:1",
|
|
"namespace": test_namespace,
|
|
},
|
|
headers=api_headers,
|
|
timeout=15,
|
|
)
|
|
assert r.status_code >= 400
|
|
assert "not allowed" in r.text.lower() or "registry" in r.text.lower()
|
|
|
|
def test_public_image_create_still_works(
|
|
self, api_base_url: str, api_headers: dict, test_namespace: str, cleanup_sandbox
|
|
):
|
|
name = "ssrf-public-ok"
|
|
cleanup_sandbox(name, test_namespace)
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={"name": name, "image": "alpine:3.21", "namespace": test_namespace},
|
|
headers=api_headers,
|
|
timeout=30,
|
|
)
|
|
assert r.status_code == 201, f"create failed: {r.text}"
|
|
_wait_for_sandbox_ready(api_base_url, api_headers, name, test_namespace)
|
|
|
|
|
|
class TestApiNamespaceAuthz:
|
|
"""Scoped API keys may only operate in their listed namespaces."""
|
|
|
|
def test_scoped_key_alpha_ok_beta_forbidden(self, api_base_url: str, make_namespace, cleanup_sandbox):
|
|
alpha, beta = make_namespace("alpha"), make_namespace("beta")
|
|
scoped_name, unscoped_name = f"integ-scoped-{alpha}", f"integ-unscoped-{alpha}"
|
|
try:
|
|
scoped_headers = {"X-API-Key": _generate_test_api_key(name=scoped_name, namespaces=[alpha])}
|
|
unscoped_headers = {"X-API-Key": _generate_test_api_key(name=unscoped_name)}
|
|
|
|
name = "ns-authz-sb"
|
|
cleanup_sandbox(name, alpha)
|
|
cleanup_sandbox(name, beta)
|
|
|
|
# Scoped key: create in alpha OK
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={"name": name, "image": "alpine:3.21", "namespace": alpha},
|
|
headers=scoped_headers,
|
|
timeout=30,
|
|
)
|
|
assert r.status_code == 201, f"scoped create alpha failed: {r.text}"
|
|
|
|
# List/delete in alpha OK
|
|
lr = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
params={"namespace": alpha},
|
|
headers=scoped_headers,
|
|
timeout=10,
|
|
)
|
|
assert lr.status_code == 200
|
|
|
|
# beta denied
|
|
br = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
params={"namespace": beta},
|
|
headers=scoped_headers,
|
|
timeout=10,
|
|
)
|
|
assert br.status_code == 403
|
|
|
|
br2 = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={"name": name, "image": "alpine:3.21", "namespace": beta},
|
|
headers=scoped_headers,
|
|
timeout=15,
|
|
)
|
|
assert br2.status_code == 403
|
|
|
|
# all-namespaces list denied
|
|
ar = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
headers=scoped_headers,
|
|
timeout=10,
|
|
)
|
|
assert ar.status_code == 403
|
|
|
|
# Unscoped key still works on beta
|
|
ur = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
params={"namespace": beta},
|
|
headers=unscoped_headers,
|
|
timeout=10,
|
|
)
|
|
assert ur.status_code == 200
|
|
|
|
# Cleanup via scoped key in alpha
|
|
dr = httpx.delete(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}",
|
|
params={"namespace": alpha},
|
|
headers=scoped_headers,
|
|
timeout=30,
|
|
)
|
|
assert dr.status_code == 200, dr.text
|
|
finally:
|
|
_revoke_test_api_key(scoped_name)
|
|
_revoke_test_api_key(unscoped_name)
|
|
|
|
def test_scoped_key_nodes_storage_forbidden(self, api_base_url: str, api_headers: dict):
|
|
"""Cluster-scoped GET /nodes/storage is 403 for a scoped key; unscoped still 200."""
|
|
scoped_name = "integ-scoped-nodes-storage"
|
|
scoped = _generate_test_api_key(name=scoped_name, namespaces=["alpha"])
|
|
try:
|
|
scoped_r = httpx.get(
|
|
f"{api_base_url}/api/v1/nodes/storage",
|
|
headers={"X-API-Key": scoped},
|
|
timeout=30,
|
|
)
|
|
assert scoped_r.status_code == 403, scoped_r.text
|
|
assert "all-namespaces" in scoped_r.json()["error"]["message"]
|
|
|
|
unscoped_r = httpx.get(
|
|
f"{api_base_url}/api/v1/nodes/storage",
|
|
headers=api_headers,
|
|
timeout=120,
|
|
)
|
|
assert unscoped_r.status_code == 200, unscoped_r.text
|
|
data = unscoped_r.json()["data"]
|
|
assert isinstance(data, dict) and data, f"expected per-node map, got {data}"
|
|
finally:
|
|
_revoke_test_api_key(scoped_name)
|
|
|
|
|
|
class TestApiNodeAuthz:
|
|
"""Node-scoped API keys may only place sandboxes on listed nodes."""
|
|
|
|
def test_single_node_key_auto_pins_and_rejects_other(
|
|
self,
|
|
api_base_url: str,
|
|
test_namespace: str,
|
|
cleanup_sandbox,
|
|
):
|
|
local = os.uname().nodename
|
|
scoped_name = "integ-scoped-node"
|
|
scoped = _generate_test_api_key(name=scoped_name, nodes=[local])
|
|
scoped_headers = {"X-API-Key": scoped}
|
|
name = "node-authz-sb"
|
|
cleanup_sandbox(name, test_namespace)
|
|
try:
|
|
r = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={"name": name, "image": "alpine:3.21", "namespace": test_namespace},
|
|
headers=scoped_headers,
|
|
timeout=180,
|
|
)
|
|
assert r.status_code == 201, r.text
|
|
_wait_for_sandbox_ready(api_base_url, scoped_headers, name, test_namespace)
|
|
got = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes/{name}",
|
|
params={"namespace": test_namespace},
|
|
headers=scoped_headers,
|
|
timeout=10,
|
|
)
|
|
assert got.status_code == 200, got.text
|
|
assert got.json()["data"].get("node") == local
|
|
|
|
denied = httpx.post(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
json={
|
|
"name": f"{name}-x",
|
|
"image": "alpine:3.21",
|
|
"namespace": test_namespace,
|
|
"node_name": "not-a-k7-node",
|
|
},
|
|
headers=scoped_headers,
|
|
timeout=15,
|
|
)
|
|
assert denied.status_code == 403, denied.text
|
|
assert "not-a-k7-node" in denied.json()["error"]["message"]
|
|
finally:
|
|
_revoke_test_api_key(scoped_name)
|
|
|
|
def test_node_scoped_key_nodes_storage_forbidden(self, api_base_url: str):
|
|
scoped_name = "integ-scoped-nodes-only"
|
|
scoped = _generate_test_api_key(name=scoped_name, nodes=[os.uname().nodename])
|
|
try:
|
|
scoped_r = httpx.get(
|
|
f"{api_base_url}/api/v1/nodes/storage",
|
|
headers={"X-API-Key": scoped},
|
|
timeout=30,
|
|
)
|
|
assert scoped_r.status_code == 403, scoped_r.text
|
|
assert "cluster-wide node" in scoped_r.json()["error"]["message"]
|
|
finally:
|
|
_revoke_test_api_key(scoped_name)
|
|
|
|
|
|
def _kubectl_out(*args: str) -> str:
|
|
r = subprocess.run([*KUBECTL, *args], capture_output=True, text=True)
|
|
assert r.returncode == 0, r.stderr
|
|
return r.stdout
|
|
|
|
|
|
class TestApiSecurityHardening:
|
|
"""Acceptance checks that are safe to automate on a shared node."""
|
|
|
|
def test_authenticate_never_writes_key_secret(self, api_base_url: str, api_headers: dict):
|
|
rv = "jsonpath={.metadata.resourceVersion}"
|
|
before = _kubectl_out("-n", system_ns(), "get", "secret", "k7-api-keys", "-o", rv)
|
|
with httpx.Client() as client:
|
|
for _ in range(200):
|
|
assert (
|
|
client.get(f"{api_base_url}/api/v1/sandboxes", headers=api_headers, timeout=10).status_code == 200
|
|
)
|
|
assert _kubectl_out("-n", system_ns(), "get", "secret", "k7-api-keys", "-o", rv) == before
|
|
|
|
def test_revoke_takes_effect_within_10s(self, api_base_url: str):
|
|
name = f"revoke-{int(time.time())}"
|
|
token = _generate_test_api_key(name=name)
|
|
try:
|
|
_revoke_test_api_key(name)
|
|
time.sleep(10)
|
|
for _ in range(10): # spread across both replicas
|
|
r = httpx.get(f"{api_base_url}/api/v1/sandboxes", headers={"X-API-Key": token}, timeout=5)
|
|
assert r.status_code == 401
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
def test_mutating_request_audited_without_token(self, api_base_url: str):
|
|
name = f"audit-{int(time.time())}"
|
|
token = _generate_test_api_key(name=name)
|
|
try:
|
|
r = httpx.delete(
|
|
f"{api_base_url}/api/v1/sandboxes/audit-nonexistent",
|
|
params={"namespace": "default"},
|
|
headers={"X-API-Key": token},
|
|
timeout=30,
|
|
)
|
|
request_id = r.headers["X-Request-ID"]
|
|
logs = _kubectl_out("-n", system_ns(), "logs", "-l", "app=k7-api", "-c", "k7-api", "--tail=2000")
|
|
lines = [json.loads(line) for line in logs.splitlines() if request_id in line]
|
|
lines = [line for line in lines if line.get("logger") == "k7.audit"]
|
|
assert len(lines) == 1, lines
|
|
assert lines[0]["key_name"] == name
|
|
assert lines[0]["route"] == "/api/v1/sandboxes/{name}"
|
|
assert lines[0]["method"] == "DELETE"
|
|
assert token not in logs
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
def test_auth_failures_are_audited_on_reads(self, api_base_url: str):
|
|
"""401 and 403 are audited even for GET, with the key id but never the token."""
|
|
unknown = "k7_abcdefgh_" + "A" * 43
|
|
r401 = httpx.get(f"{api_base_url}/api/v1/sandboxes", headers={"X-API-Key": unknown}, timeout=5)
|
|
assert r401.status_code == 401
|
|
name = f"audit-scoped-{int(time.time())}"
|
|
token = _generate_test_api_key(name=name, namespaces=["alpha"])
|
|
try:
|
|
r403 = httpx.get(
|
|
f"{api_base_url}/api/v1/sandboxes",
|
|
params={"namespace": "beta"},
|
|
headers={"X-API-Key": token},
|
|
timeout=5,
|
|
)
|
|
assert r403.status_code == 403
|
|
logs = _kubectl_out("-n", system_ns(), "logs", "-l", "app=k7-api", "-c", "k7-api", "--tail=2000")
|
|
by_id = {}
|
|
for r in (r401, r403):
|
|
rid = r.headers["X-Request-ID"]
|
|
found = [json.loads(line) for line in logs.splitlines() if rid in line]
|
|
# kubectl merges stdout and stderr. The access line (k7.api) carries the
|
|
# same request id; only the k7.audit line is the audit entry.
|
|
found = [line for line in found if line.get("logger") == "k7.audit"]
|
|
assert len(found) == 1, (rid, found)
|
|
by_id[r.status_code] = found[0]
|
|
assert by_id[401]["key_id"] == "abcdefgh" and by_id[401]["key_name"] is None
|
|
assert by_id[401]["method"] == "GET" and by_id[401]["status"] == 401
|
|
assert by_id[403]["key_name"] == name and by_id[403]["status"] == 403
|
|
assert unknown not in logs and token not in logs
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
def test_legacy_key_in_secret_authenticates(self, api_base_url: str):
|
|
"""A migrated bare token (``legacy-<sha256[:8]>`` entry, given its role the way
|
|
``k7 api keys migrate`` does) is matched by hash scan."""
|
|
import secrets
|
|
|
|
from k7.deploy.k7_api_keys import KEYS_SECRET, legacy_entries, migrate_roles, sha256_hex
|
|
|
|
token = secrets.token_hex(24)
|
|
name = f"integ-legacy-{int(time.time())}"
|
|
now = int(time.time())
|
|
legacy = {"name": name, "created": now, "expires": now + 3600, "admin": True}
|
|
entries = legacy_entries({sha256_hex(token): legacy})
|
|
assert list(entries) == [f"legacy-{sha256_hex(token)[:8]}"]
|
|
roles, _ = migrate_roles(entries)
|
|
assert json.loads(roles[f"legacy-{sha256_hex(token)[:8]}"])["role"] == "admin"
|
|
try:
|
|
_keys_kubectl().patch(KEYS_SECRET, roles)
|
|
_wait_key_live(token)
|
|
for _ in range(4): # both replicas
|
|
r = httpx.get(f"{api_base_url}/api/v1/sandboxes", headers={"X-API-Key": token}, timeout=5)
|
|
assert r.status_code == 200, r.text
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
def test_health_reports_a_synced_secret_store(self, api_base_url: str):
|
|
from k7.api.keystore import POLL_SECONDS
|
|
|
|
for _ in range(4):
|
|
store = httpx.get(f"{api_base_url}/health", timeout=5).json()["key_store"]
|
|
assert store["store"] == "secret" and store["sync_error"] is None, store
|
|
assert store["synced_age_s"] is not None and store["synced_age_s"] < 3 * POLL_SECONDS, store
|
|
|
|
@pytest.mark.skipif(
|
|
os.environ.get("K7_ITEST_SLOW") != "1", reason="waits one usage flush period (~5 min); set K7_ITEST_SLOW=1"
|
|
)
|
|
def test_last_used_is_flushed_to_usage_secret(self, api_base_url: str):
|
|
from k7.api.keystore import FLUSH_SECONDS
|
|
from k7.deploy.k7_api_keys import USAGE_SECRET, token_key_id
|
|
|
|
name = f"usage-{int(time.time())}"
|
|
token = _generate_test_api_key(name=name)
|
|
key_id = token_key_id(token)
|
|
assert key_id
|
|
try:
|
|
used_at = int(time.time())
|
|
assert httpx.get(f"{api_base_url}/api/v1/sandboxes", headers={"X-API-Key": token}, timeout=5).is_success
|
|
deadline = time.monotonic() + FLUSH_SECONDS + 60
|
|
while time.monotonic() < deadline:
|
|
if int((_keys_kubectl().read(USAGE_SECRET) or {}).get(key_id) or 0) >= used_at:
|
|
return
|
|
time.sleep(10)
|
|
raise AssertionError(f"last_used of {key_id} not in Secret {USAGE_SECRET} within {FLUSH_SECONDS + 60}s")
|
|
finally:
|
|
_revoke_test_api_key(name)
|
|
|
|
def test_api_and_agent_run_one_pinned_image(self):
|
|
"""One image for k7-api and k7-agent: a digest pulled IfNotPresent, or the
|
|
locally built k7-api:local with Never."""
|
|
from k7.deploy.images import DIGEST_REF, LOCAL_API_IMAGE_REF
|
|
|
|
def containers(app: str, name: str) -> list[dict]:
|
|
pods = json.loads(_kubectl_out("-n", system_ns(), "get", "pods", "-l", f"app={app}", "-o", "json"))
|
|
found = [c for p in pods["items"] for c in p["spec"]["containers"] if c["name"] == name]
|
|
assert found, f"no {name} containers in app={app} pods"
|
|
return found
|
|
|
|
api = containers("k7-api", "k7-api")
|
|
images = {c["image"] for c in api}
|
|
assert len(images) == 1, images
|
|
image = images.pop()
|
|
policy = "Never" if image == LOCAL_API_IMAGE_REF else "IfNotPresent"
|
|
assert image == LOCAL_API_IMAGE_REF or DIGEST_REF.match(image), image
|
|
for c in [*api, *containers("k7-agent", "k7-agent")]:
|
|
assert (c["image"], c["imagePullPolicy"]) == (image, policy)
|