mirror of
https://github.com/Katakate/k7.git
synced 2026-11-19 02:17:10 +00:00
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
780 lines
31 KiB
Python
780 lines
31 KiB
Python
"""Node lease, node reset and hardware-e2e scripts, driven with fakes on PATH.
|
|
|
|
Nothing here reaches a network: ``ssh``, ``scp``, ``rsync``, ``curl`` and
|
|
``sleep`` are fakes that log their calls. In "exec" mode the fake ssh runs the
|
|
remote script locally with node paths rewritten into a temp dir; in "script"
|
|
mode it answers from canned rules. All addresses are RFC 5737 documentation IPs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
UTILS = REPO / "utils"
|
|
IP1, IP2 = "192.0.2.11", "192.0.2.12"
|
|
|
|
pytestmark = [
|
|
pytest.mark.skipif(not (UTILS / "node-lease.sh").exists(), reason="lease/CI tooling is not part of this tree"),
|
|
pytest.mark.skipif(
|
|
not all(shutil.which(c) for c in ("bash", "jq", "flock", "python3")), reason="needs bash, jq, flock"
|
|
),
|
|
]
|
|
|
|
FAKE_SSH = r"""#!/usr/bin/env python3
|
|
import json, os, re, subprocess, sys
|
|
argv = sys.argv[1:]
|
|
i = 0
|
|
while i < len(argv) and argv[i].startswith("-"):
|
|
i += 2 if argv[i] in ("-i", "-o", "-p", "-l", "-F") else 1
|
|
host, cmd = argv[i], " ".join(argv[i + 1:])
|
|
opts = argv[:i]
|
|
stdin = sys.stdin.read()
|
|
state = os.environ["FAKE_STATE"]
|
|
with open(os.path.join(state, "ssh.log"), "a") as f:
|
|
f.write(json.dumps({"host": host, "cmd": cmd, "opts": opts, "stdin": stdin}) + "\n")
|
|
if os.environ.get("FAKE_SSH_MODE") == "exec":
|
|
for src, dst in json.loads(os.environ.get("FAKE_REWRITE", "{}")).items():
|
|
cmd, stdin = cmd.replace(src, dst), stdin.replace(src, dst)
|
|
sys.exit(subprocess.run(["bash", "-c", cmd], input=stdin, text=True).returncode)
|
|
if "bash /root/run-reset.sh" in cmd:
|
|
m = re.search(r"LEASE_DATA='([^']*)'", cmd)
|
|
open(os.path.join(state, "written-lease"), "w").write(m.group(1))
|
|
rules = json.load(open(os.path.join(state, "ssh-rules.json"))) if os.path.exists(os.path.join(state, "ssh-rules.json")) else []
|
|
for pattern, out, rc in rules:
|
|
if re.search(pattern, cmd):
|
|
if out == "@lease":
|
|
out = open(os.path.join(state, "written-lease")).read() + "\n"
|
|
sys.stdout.write(out)
|
|
sys.exit(rc)
|
|
"""
|
|
|
|
FAKE_CURL = r"""#!/usr/bin/env python3
|
|
import json, os, re, sys
|
|
state = os.environ["FAKE_STATE"]
|
|
args = sys.argv[1:]
|
|
with open(os.path.join(state, "curl.log"), "a") as f:
|
|
f.write(json.dumps(args) + "\n")
|
|
url = next(a for a in args if a.startswith("https://"))
|
|
for pattern, out, rc in json.load(open(os.path.join(state, "curl-rules.json"))):
|
|
if re.search(pattern, url):
|
|
sys.stdout.write(out)
|
|
sys.exit(rc)
|
|
sys.exit(22)
|
|
"""
|
|
|
|
LOGGER = '#!/usr/bin/env bash\necho "$*" >> "$FAKE_STATE/{name}.log"\n'
|
|
FAKE_SLEEP = "#!/usr/bin/env python3\nimport time\ntime.sleep(0.02)\n"
|
|
|
|
STUB_LEASE = """#!/usr/bin/env bash
|
|
echo "$1 $2 ttl=${K7_LEASE_TTL:-} owner=${K7_LEASE_OWNER:-}" >> "$FAKE_STATE/lease.log"
|
|
f="$FAKE_STATE/lease-rc-$1-$2"; [[ -f $f ]] && exit "$(cat "$f")"; exit 0
|
|
"""
|
|
STUB_RESET = """#!/usr/bin/env bash
|
|
echo "$1 $2 node=${K7_NODE_IP:-} host=${HOSTNAME_VALUE:-} two_disk=${TWO_DISK:-}" >> "$FAKE_STATE/reset.log"
|
|
f="$FAKE_STATE/reset-rc-$2"; [[ -f $f ]] && exit "$(cat "$f")"; exit 0
|
|
"""
|
|
STUB_ITEST = """#!/usr/bin/env bash
|
|
echo "node=${K7_NODE_IP:-} artifacts=${K7_ITEST_ARTIFACTS:-} kube=${K7_KUBECONFIG:-} ns=${K7_SYSTEM_NAMESPACE:-}\
|
|
marker=${K7_ITEST_MARKER:-} run=${K7_ITEST_RUN:-} destructive=${K7_DESTRUCTIVE:-}\
|
|
install=${K7_ITEST_INSTALL_ARGS:-}" >> "$FAKE_STATE/itest.log"
|
|
"""
|
|
STUB_DETACHED = """remote_detached() {
|
|
printf '%s|%s|%s\\n' "$1" "$2" "$NODE" >> "$FAKE_STATE/detached.log"
|
|
return "$(cat "$FAKE_STATE/detached-rc" 2>/dev/null || echo 0)"
|
|
}
|
|
"""
|
|
|
|
|
|
class Fakes:
|
|
def __init__(self, tmp: Path):
|
|
self.tmp, self.bin, self.state, self.home = tmp, tmp / "bin", tmp / "state", tmp / "home"
|
|
for d in (self.bin, self.state, self.home / ".ssh"):
|
|
d.mkdir(parents=True)
|
|
(self.home / ".ssh/id_ed25519").write_text("fake private key\n")
|
|
(self.home / ".ssh/id_ed25519.pub").write_text("ssh-ed25519 AAAAfake test@k7\n")
|
|
self._write("ssh", FAKE_SSH)
|
|
self._write("curl", FAKE_CURL)
|
|
self._write("sleep", FAKE_SLEEP)
|
|
for name in ("scp", "rsync"):
|
|
self._write(name, LOGGER.format(name=name))
|
|
|
|
def _write(self, name: str, text: str, where: Path | None = None) -> Path:
|
|
p = (where or self.bin) / name
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(text)
|
|
p.chmod(0o755)
|
|
return p
|
|
|
|
def tree(self, *scripts: str, stubs: dict[str, str] | None = None) -> Path:
|
|
"""A copy of utils/ holding ``scripts`` (real), the key helper (real) and ``stubs``."""
|
|
utils = self.tmp / "repo" / "utils"
|
|
for rel in (*scripts, "ssh-key-helper.sh"):
|
|
(utils / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(UTILS / rel, utils / rel)
|
|
for rel, text in (stubs or {}).items():
|
|
self._write(rel, text, utils)
|
|
return utils
|
|
|
|
def env(self, **extra: str) -> dict[str, str]:
|
|
env = {
|
|
"PATH": f"{self.bin}:/usr/local/bin:/usr/bin:/bin",
|
|
"HOME": str(self.home),
|
|
"FAKE_STATE": str(self.state),
|
|
"LANG": "C.UTF-8",
|
|
}
|
|
env.update(extra)
|
|
assert shutil.which("ssh", path=env["PATH"]) == str(self.bin / "ssh")
|
|
assert shutil.which("curl", path=env["PATH"]) == str(self.bin / "curl")
|
|
return env
|
|
|
|
def run(self, argv: list[str], env: dict[str, str], timeout: int = 120) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["bash", *argv], capture_output=True, text=True, env=env, stdin=subprocess.DEVNULL, timeout=timeout
|
|
)
|
|
|
|
def rules(self, name: str, rules: list[tuple[str, str, int]]) -> None:
|
|
(self.state / f"{name}-rules.json").write_text(json.dumps(rules))
|
|
|
|
def log(self, name: str) -> list[str]:
|
|
p = self.state / f"{name}.log"
|
|
return p.read_text().splitlines() if p.exists() else []
|
|
|
|
def ssh_calls(self) -> list[dict]:
|
|
return [json.loads(line) for line in self.log("ssh")]
|
|
|
|
|
|
@pytest.fixture()
|
|
def fakes(tmp_path: Path) -> Fakes:
|
|
return Fakes(tmp_path)
|
|
|
|
|
|
# --- node-lease.sh -------------------------------------------------------------
|
|
|
|
|
|
class Node:
|
|
"""node-lease.sh with its remote half run locally against a fake node root."""
|
|
|
|
def __init__(self, fakes: Fakes, os_id: str = "ubuntu"):
|
|
self.fakes, self.root = fakes, fakes.tmp / "node"
|
|
(self.root / "etc").mkdir(parents=True)
|
|
(self.root / "etc/os-release").write_text(f'NAME="X"\nID={os_id}\n')
|
|
self.script = fakes.tree("node-lease.sh") / "node-lease.sh"
|
|
self.rewrite = {
|
|
"/var/lib/k7-lease": str(self.root / "var/lib/k7-lease"),
|
|
"/etc/os-release": str(self.root / "etc/os-release"),
|
|
}
|
|
self.lease = self.root / "var/lib/k7-lease/lease"
|
|
|
|
def __call__(self, action: str, owner: str | None = "agent:a", **env: str) -> subprocess.CompletedProcess[str]:
|
|
e = {"FAKE_SSH_MODE": "exec", "FAKE_REWRITE": json.dumps(self.rewrite), **env}
|
|
if owner is not None:
|
|
e["K7_LEASE_OWNER"] = owner
|
|
return self.fakes.run([str(self.script), action, IP1], self.fakes.env(**e))
|
|
|
|
def put(self, owner: str, expires_in: int) -> None:
|
|
self.lease.parent.mkdir(parents=True, exist_ok=True)
|
|
now = int(time.time())
|
|
self.lease.write_text(f"owner={owner}\nexpires={now + expires_in}\nacquired={now}\n")
|
|
|
|
def fields(self) -> dict[str, str]:
|
|
return dict(line.split("=", 1) for line in self.lease.read_text().splitlines())
|
|
|
|
|
|
def test_lease_acquire_writes_owner_and_expiry(fakes: Fakes):
|
|
node = Node(fakes)
|
|
before = int(time.time())
|
|
r = node("acquire", K7_LEASE_TTL="600")
|
|
assert r.returncode == 0, r.stderr
|
|
f = node.fields()
|
|
assert f["owner"] == "agent:a"
|
|
assert before + 600 <= int(f["expires"]) <= int(time.time()) + 600
|
|
(call,) = fakes.ssh_calls()
|
|
assert call["host"] == f"root@{IP1}"
|
|
assert call["cmd"] == "bash -s -- acquire agent:a 600"
|
|
assert "BatchMode=yes" in call["opts"] and "ConnectTimeout=10" in call["opts"]
|
|
|
|
|
|
def test_lease_acquire_respects_a_live_lease_and_takes_an_expired_one(fakes: Fakes):
|
|
node = Node(fakes)
|
|
node.put("ci:other#1", 3600)
|
|
r = node("acquire")
|
|
assert r.returncode == 3 and "held by ci:other#1" in r.stderr
|
|
assert node.fields()["owner"] == "ci:other#1"
|
|
node.put("ci:other#1", -1)
|
|
assert node("acquire").returncode == 0
|
|
assert node.fields()["owner"] == "agent:a"
|
|
|
|
|
|
def test_lease_reacquire_by_the_owner_extends_it(fakes: Fakes):
|
|
node = Node(fakes)
|
|
node.put("agent:a", 10)
|
|
assert node("acquire", K7_LEASE_TTL="5000").returncode == 0
|
|
assert int(node.fields()["expires"]) >= int(time.time()) + 4990
|
|
|
|
|
|
def test_lease_acquire_refuses_outside_ubuntu(fakes: Fakes):
|
|
node = Node(fakes, os_id="debian")
|
|
r = node("acquire")
|
|
assert r.returncode == 3 and "not booted into Ubuntu" in r.stderr
|
|
assert not node.lease.exists()
|
|
|
|
|
|
def test_lease_renew_only_by_the_owner(fakes: Fakes):
|
|
node = Node(fakes)
|
|
node.put("ci:other#1", 3600)
|
|
r = node("renew")
|
|
assert r.returncode == 3 and "not held by agent:a (owner: ci:other#1)" in r.stderr
|
|
node.put("agent:a", 5)
|
|
assert node("renew", K7_LEASE_TTL="900").returncode == 0
|
|
assert int(node.fields()["expires"]) >= int(time.time()) + 890
|
|
|
|
|
|
def test_lease_release_rules(fakes: Fakes):
|
|
node = Node(fakes)
|
|
r = node("release")
|
|
assert r.returncode == 0 and "lease already free" in r.stdout
|
|
node.put("ci:other#1", 3600)
|
|
r = node("release")
|
|
assert r.returncode == 3 and node.lease.exists()
|
|
node.put("ci:other#1", -5)
|
|
r = node("release")
|
|
assert r.returncode == 0 and "expired lease of ci:other#1 left as is" in r.stdout and node.lease.exists()
|
|
node.put("agent:a", 3600)
|
|
r = node("release")
|
|
assert r.returncode == 0 and not node.lease.exists()
|
|
|
|
|
|
def test_lease_check_needs_an_unexpired_lease_of_ours(fakes: Fakes):
|
|
node = Node(fakes)
|
|
assert node("check").returncode == 3
|
|
node.put("agent:a", 3600)
|
|
r = node("check")
|
|
assert r.returncode == 0 and "lease held by agent:a" in r.stdout
|
|
node.put("agent:a", -1)
|
|
assert node("check").returncode == 3
|
|
node.put("ci:other#1", 3600)
|
|
r = node("check")
|
|
assert r.returncode == 3 and "owner: ci:other#1" in r.stderr
|
|
|
|
|
|
def test_lease_status_and_break_need_no_owner(fakes: Fakes):
|
|
node = Node(fakes)
|
|
assert node("status", owner=None).stdout.strip() == "free"
|
|
node.put("ci:other#1", 3600)
|
|
assert node("status", owner=None).stdout.startswith("held: owner=ci:other#1 expires=")
|
|
node.put("ci:other#1", -1)
|
|
assert node("status", owner=None).stdout.strip() == "expired: owner=ci:other#1"
|
|
r = node("break", owner=None)
|
|
assert r.returncode == 0 and "was: ci:other#1" in r.stdout and not node.lease.exists()
|
|
assert fakes.ssh_calls()[0]["cmd"] == "bash -s -- status - 14400"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("env", "msg"),
|
|
[
|
|
({"K7_LEASE_TTL": "4h"}, "K7_LEASE_TTL must be seconds"),
|
|
({"K7_LEASE_OWNER": "agent a"}, "must not contain spaces or '='"),
|
|
({"K7_LEASE_OWNER": "owner=x"}, "must not contain spaces or '='"),
|
|
],
|
|
)
|
|
def test_lease_bad_input_exits_2_before_ssh(fakes: Fakes, env: dict[str, str], msg: str):
|
|
node = Node(fakes)
|
|
r = node("acquire", **{"owner": "agent:a", **{k.replace("K7_LEASE_OWNER", "owner"): v for k, v in env.items()}})
|
|
assert r.returncode == 2 and msg in r.stderr
|
|
assert fakes.ssh_calls() == []
|
|
|
|
|
|
def test_lease_owner_required_except_for_status(fakes: Fakes):
|
|
node = Node(fakes)
|
|
r = node("acquire", owner=None)
|
|
assert r.returncode != 0 and "set K7_LEASE_OWNER" in r.stderr
|
|
assert fakes.ssh_calls() == []
|
|
|
|
|
|
def test_lease_unknown_action_is_2(fakes: Fakes):
|
|
node = Node(fakes)
|
|
r = node("steal")
|
|
assert r.returncode == 2 and "unknown action: steal" in r.stderr
|
|
|
|
|
|
def test_lease_busy_lock_is_4(fakes: Fakes):
|
|
node = Node(fakes)
|
|
node.rewrite["flock -w 30"] = "flock -w 0"
|
|
lock = node.root / "var/lib/k7-lease/.lock"
|
|
lock.parent.mkdir(parents=True)
|
|
with open(lock, "w") as fd:
|
|
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
r = node("acquire")
|
|
assert r.returncode == 4 and "lease lock busy" in r.stderr
|
|
|
|
|
|
# --- reset_robot_node.sh ---------------------------------------------------------
|
|
|
|
RESCUE_BOOT, UBUNTU_BOOT = "11111111-rescue", "22222222-ubuntu"
|
|
|
|
|
|
def _reset_setup(fakes: Fakes, **ssh_overrides: tuple[str, int]) -> Path:
|
|
utils = fakes.tree("reset_robot_node.sh", stubs={"node-lease.sh": STUB_LEASE, "wipe-disk.sh": "#!/bin/sh\n"})
|
|
fakes.rules(
|
|
"curl",
|
|
[
|
|
(r"/key$", json.dumps([{"key": {"fingerprint": "aa:bb"}}]), 0),
|
|
(r"/boot/\d+/rescue$", json.dumps({"rescue": {"password": "SECRET-PW", "active": True}}), 0),
|
|
(r"/reset/\d+$", json.dumps({"reset": {"type": "hw"}}), 0),
|
|
],
|
|
)
|
|
defaults = {
|
|
"reboot": ("", 255),
|
|
"sshd -T": ("", 0),
|
|
"^hostname$": ("k7-node-07\n", 0),
|
|
"cat /var/lib/k7-lease/lease": ("@lease", 0),
|
|
'grep -q "\\^ID=ubuntu"': (UBUNTU_BOOT + "\n", 0),
|
|
"^cat /proc/sys/kernel/random/boot_id$": (RESCUE_BOOT + "\n", 0),
|
|
}
|
|
defaults.update(ssh_overrides)
|
|
fakes.rules("ssh", [(p, out, rc) for p, (out, rc) in defaults.items()])
|
|
return utils / "reset_robot_node.sh"
|
|
|
|
|
|
def _reset(fakes: Fakes, script: Path, **env: str) -> subprocess.CompletedProcess[str]:
|
|
base = {
|
|
"ROBOT_USER": "u",
|
|
"ROBOT_PASS": "p",
|
|
"K7_LEASE_OWNER": "agent:a",
|
|
"K7_LEASE_TTL": "7200",
|
|
"HOSTNAME_VALUE": "k7-node-07",
|
|
}
|
|
return fakes.run([str(script), "424242", IP1], fakes.env(**{**base, **env}))
|
|
|
|
|
|
def test_reset_happy_path_writes_the_lease_into_the_new_os(fakes: Fakes):
|
|
script = _reset_setup(fakes)
|
|
before = int(time.time())
|
|
r = _reset(fakes, script, TWO_DISK="1")
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
assert "==> sshd is key-only" in r.stdout
|
|
assert "SECRET-PW" not in r.stdout
|
|
assert fakes.log("lease") == [f"check {IP1} ttl=7200 owner=agent:a"]
|
|
curls = [json.loads(line) for line in fakes.log("curl")]
|
|
assert [next(a for a in c if a.startswith("https://")).rsplit("/", 2)[-2:] for c in curls] == [
|
|
["robot-ws.your-server.de", "key"],
|
|
["424242", "rescue"],
|
|
["reset", "424242"],
|
|
]
|
|
assert "authorized_key[]=aa:bb" in curls[1] and "type=hw" in curls[2]
|
|
run = next(c["cmd"] for c in fakes.ssh_calls() if "bash /root/run-reset.sh" in c["cmd"])
|
|
assert "TWO_DISK='1'" in run and "HOSTNAME_VALUE='k7-node-07'" in run
|
|
lease = dict(line.split("=", 1) for line in (fakes.state / "written-lease").read_text().splitlines())
|
|
assert lease["owner"] == "agent:a"
|
|
assert before + 7200 <= int(lease["expires"]) <= int(time.time()) + 7200
|
|
assert all(c["host"] == f"root@{IP1}" for c in fakes.ssh_calls())
|
|
assert fakes.log("scp") and fakes.log("scp")[0].endswith(f"root@{IP1}:/root/wipe-disk.sh")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("env", "msg"),
|
|
[
|
|
({"K7_LEASE_OWNER": ""}, "set K7_LEASE_OWNER"),
|
|
({"K7_LEASE_OWNER": "agent a"}, "must have no spaces"),
|
|
({"K7_LEASE_OWNER": "agent'a"}, "must have no spaces"),
|
|
({"K7_LEASE_TTL": "2h"}, "K7_LEASE_TTL must be seconds"),
|
|
({"ROBOT_PASS": ""}, "set ROBOT_PASS"),
|
|
],
|
|
)
|
|
def test_reset_refuses_bad_input_before_touching_anything(fakes: Fakes, env: dict[str, str], msg: str):
|
|
script = _reset_setup(fakes)
|
|
r = _reset(fakes, script, **env)
|
|
assert r.returncode != 0 and msg in r.stderr
|
|
assert fakes.log("curl") == [] and fakes.ssh_calls() == [] and fakes.log("lease") == []
|
|
|
|
|
|
def test_reset_needs_the_callers_lease(fakes: Fakes):
|
|
script = _reset_setup(fakes)
|
|
(fakes.state / f"lease-rc-check-{IP1}").write_text("3")
|
|
r = _reset(fakes, script)
|
|
assert r.returncode == 3
|
|
assert fakes.log("curl") == [] and fakes.ssh_calls() == []
|
|
|
|
|
|
def test_reset_of_an_unreachable_node_skips_the_lease_check(fakes: Fakes):
|
|
script = _reset_setup(fakes)
|
|
(fakes.state / f"lease-rc-check-{IP1}").write_text("3")
|
|
r = _reset(fakes, script, K7_LEASE_NODE_UNREACHABLE="1")
|
|
assert r.returncode == 0, r.stderr
|
|
assert fakes.log("lease") == [] and "not verifying the lease" in r.stdout
|
|
|
|
|
|
def test_reset_without_a_robot_key_stops_before_rescue(fakes: Fakes):
|
|
script = _reset_setup(fakes)
|
|
fakes.rules("curl", [(r"/key$", "[]", 0)])
|
|
r = _reset(fakes, script)
|
|
assert r.returncode == 1 and "Could not get SSH key fingerprint" in r.stderr
|
|
assert len(fakes.log("curl")) == 1 and fakes.ssh_calls() == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("override", "msg"),
|
|
[
|
|
({"reboot": ("", 1)}, "reboot command failed with exit 1"),
|
|
({'grep -q "\\^ID=ubuntu"': (RESCUE_BOOT + "\n", 0)}, "refusing to trust it"),
|
|
({"^hostname$": ("rescue\n", 0)}, "Fresh OS hostname is 'rescue', expected 'k7-node-07'"),
|
|
({"cat /var/lib/k7-lease/lease": ("owner=someone-else\n", 0)}, "does not match the one written"),
|
|
({"sshd -T": ("", 1)}, "still accepts passwords or keyboard-interactive"),
|
|
({"^cat /proc/sys/kernel/random/boot_id$": ("", 0)}, "Could not read rescue boot_id"),
|
|
],
|
|
)
|
|
def test_reset_refuses_to_hand_over_an_unverified_os(fakes: Fakes, override: dict, msg: str):
|
|
script = _reset_setup(fakes, **override)
|
|
r = _reset(fakes, script)
|
|
assert r.returncode == 1 and msg in r.stderr, r.stdout + r.stderr
|
|
|
|
|
|
def test_reset_reboot_exit_0_is_accepted(fakes: Fakes):
|
|
script = _reset_setup(fakes, reboot=("", 0))
|
|
assert _reset(fakes, script).returncode == 0
|
|
|
|
|
|
# --- utils/ci/k7-e2e.sh -----------------------------------------------------------
|
|
|
|
|
|
def _e2e(fakes: Fakes, nodes: str, phases: str, robot: dict[str, str] | None = None, **env: str):
|
|
utils = fakes.tree(
|
|
"ci/k7-e2e.sh",
|
|
stubs={
|
|
"node-lease.sh": STUB_LEASE,
|
|
"reset_robot_node.sh": STUB_RESET,
|
|
"run-integration-tests.sh": STUB_ITEST,
|
|
"remote-detached.sh": STUB_DETACHED,
|
|
"ci/apt-get.sh": "",
|
|
},
|
|
)
|
|
robot = robot if robot is not None else {"1": IP1, "2": IP2}
|
|
fakes.rules("curl", [(f"/server/{s}$", json.dumps({"server": {"server_ip": ip}}), 0) for s, ip in robot.items()])
|
|
fakes.rules("ssh", [("^hostname$", "k7-ci-host\n", 0)])
|
|
base = {
|
|
"K7_CI_NODES": nodes,
|
|
"K7_CI_FORBIDDEN_SERVERS": "999",
|
|
"K7_LEASE_OWNER": "ci:test#1",
|
|
"K7_CI_PHASES": phases,
|
|
"K7_CI_ARTIFACTS": str(fakes.tmp / "art"),
|
|
"ROBOT_USER": "u",
|
|
"ROBOT_PASS": "p",
|
|
}
|
|
return fakes.run([str(utils / "ci/k7-e2e.sh")], fakes.env(**{**base, **env}))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("nodes", "msg"),
|
|
[
|
|
(f"1:{IP1} 999:{IP2}", "server 999 is forbidden; refusing"),
|
|
(f"1-{IP1}", f"bad K7_CI_NODES entry: 1-{IP1}"),
|
|
(f"1:{IP1}:22", "bad K7_CI_NODES entry"),
|
|
],
|
|
)
|
|
def test_e2e_refuses_bad_or_forbidden_nodes_before_leasing(fakes: Fakes, nodes: str, msg: str):
|
|
r = _e2e(fakes, nodes, "reset")
|
|
assert r.returncode == 1 and msg in r.stderr
|
|
assert fakes.log("lease") == [] and fakes.log("reset") == []
|
|
|
|
|
|
def test_e2e_robot_mismatch_refuses_and_releases_only_its_own_leases(fakes: Fakes):
|
|
(fakes.state / f"lease-rc-check-{IP2}").write_text("3")
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "reset", robot={"1": IP1, "2": "192.0.2.99"})
|
|
assert r.returncode == 1 and f"Robot server 2 is 192.0.2.99, not {IP2}; refusing" in r.stderr
|
|
assert fakes.log("reset") == []
|
|
assert [line.split()[:2] for line in fakes.log("lease")] == [
|
|
["check", IP1],
|
|
["check", IP2],
|
|
["acquire", IP2],
|
|
["release", IP2],
|
|
]
|
|
|
|
|
|
def test_e2e_resets_every_node_in_parallel_with_numbered_hostnames(fakes: Fakes):
|
|
for ip in (IP1, IP2):
|
|
(fakes.state / f"lease-rc-check-{ip}").write_text("3")
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "reset")
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
assert sorted(fakes.log("reset")) == [
|
|
f"1 {IP1} node={IP1} host=k7-ci-01 two_disk=1",
|
|
f"2 {IP2} node={IP2} host=k7-ci-02 two_disk=1",
|
|
]
|
|
lease = fakes.log("lease")
|
|
assert f"acquire {IP1} ttl=21600 owner=ci:test#1" in lease
|
|
assert [line.split()[:2] for line in lease if line.startswith("release")] == [["release", IP1], ["release", IP2]]
|
|
assert (fakes.tmp / "art/cluster").is_dir()
|
|
|
|
|
|
def test_e2e_one_failed_reset_fails_the_run(fakes: Fakes):
|
|
(fakes.state / f"reset-rc-{IP2}").write_text("1")
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "reset")
|
|
assert r.returncode == 1
|
|
assert f"{IP1}: reset ok" in r.stdout and f"{IP2}: reset FAILED" in r.stderr
|
|
assert not [line for line in fakes.log("lease") if line.startswith("release")]
|
|
|
|
|
|
def _itest_line(art: str, **env: str) -> str:
|
|
keys = ("kube", "ns", "marker", "run", "destructive", "install")
|
|
return f"node={IP1} artifacts={art} " + " ".join(f"{k}={env.get(k, '')}" for k in keys)
|
|
|
|
|
|
def test_e2e_test_phase_runs_the_suite_on_the_first_node(fakes: Fakes):
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "test")
|
|
assert r.returncode == 0, r.stderr
|
|
assert fakes.log("itest") == [_itest_line(f"{fakes.tmp / 'art'}/itest")]
|
|
assert fakes.log("reset") == [] and fakes.log("curl") == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("env", "msg"),
|
|
[
|
|
({"K7_CI_PHASES": "reset deploy"}, "unknown phase: deploy"),
|
|
({"K7_CI_DISTRO": "kubeadm"}, "K7_CI_DISTRO must be k3s or rke2, got kubeadm"),
|
|
],
|
|
)
|
|
def test_e2e_refuses_unknown_phases_and_distros_before_leasing(fakes: Fakes, env: dict[str, str], msg: str):
|
|
r = _e2e(fakes, f"1:{IP1}", "reset", **env)
|
|
assert r.returncode == 1 and msg in r.stderr
|
|
assert fakes.log("lease") == []
|
|
|
|
|
|
def test_e2e_rke2_install_is_single_node_and_runs_the_node_script(fakes: Fakes):
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "install", K7_CI_DISTRO="rke2")
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
assert f"K7_CI_DISTRO=rke2 is single-node: using 1:{IP1} only" in r.stdout
|
|
assert {line.split()[1] for line in fakes.log("lease")} == {IP1}
|
|
assert fakes.log("detached") == [f"rke2|cd /root/k7 && deploy/helm/k7/ci/rke2-node.sh all|root@{IP1}"]
|
|
rke2_kubectl = "KUBECONFIG=/etc/rancher/rke2/rke2.yaml /var/lib/rancher/rke2/bin/kubectl"
|
|
waits = [c["cmd"] for c in fakes.ssh_calls() if "condition=Ready node" in c["cmd"]]
|
|
assert len(waits) == 1 and waits[0].startswith(f"{rke2_kubectl} wait ") and waits[0].endswith("-eq 1")
|
|
assert (fakes.tmp / "art/inventory.ini").read_text().count("ansible_host=") == 1
|
|
assert {c["host"] for c in fakes.ssh_calls()} == {f"root@{IP1}"}
|
|
|
|
|
|
def test_e2e_k3s_install_runs_k7_install_from_the_controller(fakes: Fakes):
|
|
r = _e2e(fakes, f"1:{IP1} 2:{IP2}", "install", K7_CI_INSTALL_ARGS="--api-image local --backends kfd")
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
(line,) = fakes.log("detached")
|
|
name, cmd, node = line.split("|")
|
|
assert (name, node) == ("install", f"root@{IP1}")
|
|
assert cmd.endswith("src/k7/cli/dev.sh install -i /root/k7-ci/inventory.ini --api-image local --backends kfd")
|
|
assert any(c["cmd"].startswith("k3s kubectl wait ") for c in fakes.ssh_calls())
|
|
|
|
|
|
def test_e2e_rke2_test_and_destructive_phases_point_the_suite_at_the_chart(fakes: Fakes):
|
|
r = _e2e(fakes, f"1:{IP1}", "test destructive", K7_CI_DISTRO="rke2")
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
chart = {"kube": "/etc/rancher/rke2/rke2.yaml", "ns": "k7-system"}
|
|
art = fakes.tmp / "art"
|
|
assert fakes.log("itest") == [
|
|
_itest_line(f"{art}/itest", **chart),
|
|
_itest_line(
|
|
f"{art}/destructive",
|
|
**chart,
|
|
marker="destructive",
|
|
run="destructive",
|
|
destructive="1",
|
|
install="-i /root/k7-ci/inventory.ini --api-mode development --api-image local",
|
|
),
|
|
]
|
|
assert fakes.log("rsync") == []
|
|
|
|
|
|
def test_e2e_k3s_destructive_alone_prepares_the_controller_for_the_reinstall(fakes: Fakes):
|
|
nodes = " ".join(f"{i}:192.0.2.{10 + i}" for i in (1, 2, 3))
|
|
r = _e2e(fakes, nodes, "destructive", robot={})
|
|
assert r.returncode == 0, r.stdout + r.stderr
|
|
assert fakes.log("itest") == [
|
|
_itest_line(
|
|
f"{fakes.tmp / 'art'}/destructive",
|
|
marker="destructive",
|
|
run="destructive",
|
|
destructive="1",
|
|
install="-i /root/k7-ci/inventory.ini --ha --api-mode development --api-image local",
|
|
)
|
|
]
|
|
assert len(fakes.log("rsync")) == 1
|
|
assert (fakes.tmp / "art/inventory.ini").read_text().count("ansible_host=") == 3
|
|
|
|
|
|
# --- run-integration-tests.sh -----------------------------------------------------
|
|
|
|
|
|
def _itest(fakes: Fakes, **env: str) -> subprocess.CompletedProcess[str]:
|
|
utils = fakes.tree(
|
|
"run-integration-tests.sh", stubs={"node-lease.sh": STUB_LEASE, "remote-detached.sh": STUB_DETACHED}
|
|
)
|
|
base = {"K7_NODE_IP": IP1, "K7_LEASE_OWNER": "agent:a", "K7_ITEST_ARTIFACTS": str(fakes.tmp / "art")}
|
|
return fakes.run([str(utils / "run-integration-tests.sh")], fakes.env(**{**base, **env}))
|
|
|
|
|
|
def test_itest_requires_a_lease_owner(fakes: Fakes):
|
|
r = _itest(fakes, K7_LEASE_OWNER="")
|
|
assert r.returncode != 0 and "set K7_LEASE_OWNER" in r.stderr
|
|
assert fakes.log("lease") == [] and fakes.log("rsync") == []
|
|
|
|
|
|
def test_itest_uses_a_held_lease_and_leaves_it(fakes: Fakes):
|
|
r = _itest(fakes)
|
|
assert r.returncode == 0, r.stderr
|
|
assert [line.split()[:2] for line in fakes.log("lease")] == [["check", IP1]]
|
|
|
|
|
|
def test_itest_takes_a_free_lease_for_the_run_and_releases_it_even_on_failure(fakes: Fakes):
|
|
(fakes.state / f"lease-rc-check-{IP1}").write_text("3")
|
|
(fakes.state / "detached-rc").write_text("5")
|
|
r = _itest(fakes)
|
|
assert r.returncode == 5
|
|
assert fakes.log("lease") == [
|
|
f"check {IP1} ttl= owner=agent:a",
|
|
f"acquire {IP1} ttl=10800 owner=agent:a",
|
|
f"release {IP1} ttl= owner=agent:a",
|
|
]
|
|
assert "finished with exit code 5" in r.stdout
|
|
|
|
|
|
def test_itest_a_lease_held_by_someone_else_stops_the_run(fakes: Fakes):
|
|
for action in ("check", "acquire"):
|
|
(fakes.state / f"lease-rc-{action}-{IP1}").write_text("3")
|
|
r = _itest(fakes)
|
|
assert r.returncode == 3
|
|
assert fakes.log("rsync") == [] and fakes.log("detached") == []
|
|
assert not [line for line in fakes.log("lease") if line.startswith("release")]
|
|
|
|
|
|
def test_itest_forwards_the_chart_cluster_env(fakes: Fakes):
|
|
r = _itest(
|
|
fakes,
|
|
K7_KUBECONFIG="/root/.kube/rke2 config",
|
|
K7_SYSTEM_NAMESPACE="k7-system",
|
|
K7_PYTEST_ARGS="-k egress",
|
|
)
|
|
assert r.returncode == 0, r.stderr
|
|
(line,) = fakes.log("detached")
|
|
name, cmd, node = line.split("|")
|
|
assert (name, node) == ("itest", f"root@{IP1}")
|
|
assert cmd.startswith("export K7_KUBECONFIG=/root/.kube/rke2\\ config && export K7_SYSTEM_NAMESPACE=k7-system && ")
|
|
assert "K7_API_CA" not in cmd
|
|
assert cmd.endswith("--junitxml=/root/k7-runs/itest/junit.xml -k egress")
|
|
(sync,) = fakes.log("rsync")
|
|
assert sync.endswith(f"root@{IP1}:/root/k7/")
|
|
assert [c["cmd"] for c in fakes.ssh_calls()] == ["bash -s"]
|
|
assert "uv sync --locked" in fakes.ssh_calls()[0]["stdin"]
|
|
|
|
|
|
def test_itest_destructive_run_has_its_own_marker_run_dir_and_env(fakes: Fakes):
|
|
r = _itest(
|
|
fakes,
|
|
K7_ITEST_MARKER="destructive",
|
|
K7_ITEST_RUN="destructive",
|
|
K7_DESTRUCTIVE="1",
|
|
K7_ITEST_INSTALL_ARGS="-i /root/k7-ci/inventory.ini --ha",
|
|
)
|
|
assert r.returncode == 0, r.stderr
|
|
(line,) = fakes.log("detached")
|
|
name, cmd, _ = line.split("|")
|
|
assert name == "destructive"
|
|
assert cmd.startswith(
|
|
"export K7_DESTRUCTIVE=1 && export K7_ITEST_INSTALL_ARGS=-i\\ /root/k7-ci/inventory.ini\\ --ha && "
|
|
)
|
|
assert "pytest -m destructive -v --junitxml=/root/k7-runs/destructive/junit.xml" in cmd
|
|
assert [s.split()[-2] for s in fakes.log("scp")] == [
|
|
f"root@{IP1}:/root/k7-runs/destructive/log",
|
|
f"root@{IP1}:/root/k7-runs/destructive/junit.xml",
|
|
]
|
|
|
|
|
|
def test_itest_default_marker_is_integration(fakes: Fakes):
|
|
assert _itest(fakes).returncode == 0
|
|
(line,) = fakes.log("detached")
|
|
assert "pytest -m integration -v --junitxml=/root/k7-runs/itest/junit.xml" in line
|
|
|
|
|
|
def test_itest_bad_run_name_exits_2_before_the_lease(fakes: Fakes):
|
|
r = _itest(fakes, K7_ITEST_RUN="../itest")
|
|
assert r.returncode == 2 and "K7_ITEST_RUN must match" in r.stderr
|
|
assert fakes.log("lease") == [] and fakes.log("rsync") == []
|
|
|
|
|
|
# --- remote-detached.sh -----------------------------------------------------------
|
|
|
|
|
|
def _detached(fakes: Fakes, name: str, cmd: str) -> subprocess.CompletedProcess[str]:
|
|
root = fakes.tmp / "node"
|
|
root.mkdir(exist_ok=True)
|
|
env = fakes.env(FAKE_SSH_MODE="exec", FAKE_REWRITE=json.dumps({"/root/k7-runs": str(root / "k7-runs")}))
|
|
driver = (
|
|
f"source {UTILS / 'remote-detached.sh'}\n"
|
|
f"SSH_OPTS=(-o BatchMode=yes) NODE=root@{IP1}\n"
|
|
f"remote_detached {name} {json.dumps(cmd)}\n"
|
|
)
|
|
return fakes.run(["-c", driver], env, timeout=60)
|
|
|
|
|
|
def test_detached_streams_the_log_and_returns_the_exit_status(fakes: Fakes):
|
|
r = _detached(fakes, "job", "echo first; echo second; exit 7")
|
|
assert r.returncode == 7, r.stderr
|
|
assert r.stdout.count("first") == 1 and r.stdout.count("second") == 1
|
|
assert "==> [job] exited 7" in r.stdout
|
|
run_dir = fakes.tmp / "node/k7-runs/job"
|
|
assert (run_dir / "rc").read_text().strip() == "7"
|
|
assert (run_dir / "cmd").read_text().strip() == "echo first; echo second; exit 7"
|
|
|
|
|
|
def test_detached_refuses_a_second_run_with_the_same_name(fakes: Fakes):
|
|
lock = fakes.tmp / "node/k7-runs/job.lock"
|
|
lock.parent.mkdir(parents=True)
|
|
with open(lock, "w") as fd:
|
|
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
r = _detached(fakes, "job", "echo should-not-run")
|
|
assert r.returncode == 1 and "is already running" in r.stderr
|
|
assert not (fakes.tmp / "node/k7-runs/job/cmd").exists()
|
|
|
|
|
|
# --- ci/apt-get.sh -----------------------------------------------------------------
|
|
|
|
|
|
def _apt(fakes: Fakes, wait: str) -> tuple[subprocess.CompletedProcess[str], Path]:
|
|
locks = fakes.tmp / "locks"
|
|
locks.mkdir(exist_ok=True)
|
|
text = (UTILS / "ci/apt-get.sh").read_text()
|
|
for p in ("/var/lib/apt/lists/lock", "/var/cache/apt/archives/lock", "/var/lib/dpkg/lock-frontend"):
|
|
assert p in text
|
|
text = text.replace(p, str(locks / Path(p).parent.name))
|
|
script = fakes.tmp / "apt-get.sh"
|
|
script.write_text(text)
|
|
fakes._write("apt-get", '#!/usr/bin/env bash\necho "$DEBIAN_FRONTEND $*" >> "$FAKE_STATE/apt.log"\n')
|
|
r = fakes.run([str(script), "install", "-y", "rsync"], fakes.env(K7_APT_LOCK_WAIT=wait))
|
|
return r, locks
|
|
|
|
|
|
def test_apt_wrapper_runs_apt_get_with_the_lock_timeout(fakes: Fakes):
|
|
r, _ = _apt(fakes, "120")
|
|
assert r.returncode == 0, r.stderr
|
|
assert fakes.log("apt") == ["noninteractive -o DPkg::Lock::Timeout=120 install -y rsync"]
|
|
|
|
|
|
def test_apt_wrapper_gives_up_on_a_held_lock(fakes: Fakes):
|
|
locks = fakes.tmp / "locks"
|
|
locks.mkdir()
|
|
held = locks / "lists"
|
|
held.write_text("")
|
|
fd = os.open(held, os.O_RDWR)
|
|
try:
|
|
fcntl.lockf(fd, fcntl.LOCK_EX)
|
|
r, _ = _apt(fakes, "0")
|
|
finally:
|
|
os.close(fd)
|
|
assert r.returncode != 0 and re.search(r"lists still locked after 0s", r.stderr), r.stderr
|
|
assert fakes.log("apt") == []
|