Files
G 3c80778156 Release 0.5.0
Observability, usage metering, tenants and quotas, and roles with OIDC login. A Helm chart installs K7 on a cluster you already run, API keys carry a role, and k7-agent no longer needs host privileges.
2026-10-18 23:21:48 +02:00

342 lines
12 KiB
Python

"""Integration tests for ``k7 restore``, multi-node cluster.
Coverage:
- Restore after the source sandbox has been deleted entirely (the
motivating use case).
- ``--image`` override against a legacy snapshot that lacks
``k7.io/source-image`` annotations.
- ``--no-keep-snapshot`` deletes the source snapshot after success.
- Restoring into an existing sandbox name is rejected with 409 semantics.
- Restoring from a snapshot whose ``readyToUse`` is false fails fast.
These exercise ``K7Core.restore_sandbox`` directly (async tests) so we
share an event loop with the kubernetes_asyncio clients; the
``dev.sh``-based ``test_pause_resume.py`` covers the CLI surface.
"""
import asyncio
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import pytest
from k7.core import cluster_config
from k7.core.core import K7Core
from k7.core.models import SandboxConfig, SandboxConfigOverrides
from .cluster import KUBECTL
pytestmark = pytest.mark.integration
_REPO_ROOT = Path(__file__).resolve().parents[2]
def _run_kubectl(*args: str) -> subprocess.CompletedProcess:
return subprocess.run([*KUBECTL, *args], capture_output=True, text=True)
def _wait_pod_ready(name: str, namespace: str, timeout: int = 240) -> None:
deadline = time.time() + timeout
while time.time() < deadline:
out = _run_kubectl(
"get",
"pods",
"-n",
namespace,
"-l",
f"app={name}",
"-o",
"jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}",
)
if out.returncode == 0 and out.stdout.strip() == "True":
return
time.sleep(2)
raise TimeoutError(f"pod for sandbox {name} not Ready within {timeout}s")
def _snapshot_exists(name: str, namespace: str) -> bool:
out = _run_kubectl("get", "volumesnapshot", name, "-n", namespace, "-o", "name")
return out.returncode == 0 and out.stdout.strip() != ""
def _force_delete_snapshot(name: str, namespace: str) -> None:
_run_kubectl(
"patch",
"volumesnapshot",
name,
"-n",
namespace,
"--type=merge",
"-p",
'{"metadata":{"finalizers":[]}}',
)
_run_kubectl("delete", "volumesnapshot", name, "-n", namespace, "--ignore-not-found", "--wait=false")
@pytest.fixture()
def restore_sandbox_cleanup(test_namespace: str):
"""Track sandboxes and snapshots for teardown — restore tests are messy."""
sandboxes: list[str] = []
snapshots: list[str] = []
def _register_sandbox(name: str) -> None:
sandboxes.append(name)
def _register_snapshot(name: str) -> None:
snapshots.append(name)
yield _register_sandbox, _register_snapshot
for snap in snapshots:
_force_delete_snapshot(snap, test_namespace)
for name in sandboxes:
try:
asyncio.run(K7Core().delete_sandbox(name, namespace=test_namespace))
except Exception:
pass
def _ql_config(name: str, namespace: str) -> SandboxConfig:
return SandboxConfig(
name=name,
image="alpine:3.20",
namespace=namespace,
backend="kata-qemu-longhorn",
)
class TestRestoreAfterSourceDeleted:
"""The motivating use case: pause, snapshot, delete source, restore."""
async def test_restore_after_pause_then_delete_source(
self,
test_namespace: str,
restore_sandbox_cleanup,
):
register_sandbox, register_snapshot = restore_sandbox_cleanup
source = f"restore-src-{uuid4().hex[:6]}"
snap = f"{source}-paused-keep"
restored = f"{source}-r"
register_sandbox(source)
register_sandbox(restored)
register_snapshot(snap)
k7 = K7Core()
# 1. Create + write a marker into the source's persistent state.
create = await k7.create_sandbox(_ql_config(source, test_namespace))
assert create.success, f"create failed: {create.error}"
_wait_pod_ready(source, test_namespace)
marker = await k7.exec_command(
source,
"echo restore-marker > /mnt/state/marker && cat /mnt/state/marker",
namespace=test_namespace,
)
assert marker.exit_code == 0, f"marker write failed: {marker.stderr}"
# 2. Pause + snapshot (kind=pause, persistent).
pause = await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
assert pause.success, pause.error
# Wait for the snapshot to become readyToUse before we delete the source.
deadline = time.time() + 180
ready = False
while time.time() < deadline:
info = await k7.get_snapshot(snap, namespace=test_namespace)
if info is not None and info.ready_to_use:
ready = True
break
await asyncio.sleep(2)
assert ready, f"snapshot {snap} never reached readyToUse=True"
# 3. Delete the source Deployment + PVC entirely. The snapshot is the
# only trace of the original sandbox now.
delete = await k7.delete_sandbox(source, namespace=test_namespace)
assert delete.success, delete.error
# Confirm the source's root PVC is gone.
deadline = time.time() + 60
while time.time() < deadline:
out = _run_kubectl("get", "pvc", f"{source}-root-lh", "-n", test_namespace, "-o", "name")
if out.returncode != 0 or out.stdout.strip() == "":
break
time.sleep(2)
# 4. Restore from the snapshot.
restore_k7 = K7Core() # fresh K7Core (separate event loop is fine here)
restore_result = await restore_k7.restore_sandbox(
snapshot_name=snap,
new_sandbox_name=restored,
namespace=test_namespace,
)
assert restore_result.success, f"restore failed: {restore_result.error}"
_wait_pod_ready(restored, test_namespace)
# 5. The marker survives.
check = await restore_k7.exec_command(restored, "cat /mnt/state/marker", namespace=test_namespace)
assert check.exit_code == 0
assert "restore-marker" in check.stdout
# 6. Default keep_snapshot=True → snapshot still there.
assert _snapshot_exists(snap, test_namespace)
class TestRestoreOverrides:
"""``--image`` override against an annotation-less snapshot."""
async def test_image_override_when_annotation_missing(
self,
test_namespace: str,
restore_sandbox_cleanup,
):
register_sandbox, register_snapshot = restore_sandbox_cleanup
source = f"restore-ovr-{uuid4().hex[:6]}"
snap = f"{source}-legacy"
restored = f"{source}-r"
register_sandbox(source)
register_sandbox(restored)
register_snapshot(snap)
k7 = K7Core()
create = await k7.create_sandbox(_ql_config(source, test_namespace))
assert create.success, create.error
_wait_pod_ready(source, test_namespace)
# Create a "legacy" snapshot manually: no ``k7.io/source-*`` annotations.
custom = await k7._get_custom_objects_client()
await custom.create_namespaced_custom_object(
group="snapshot.storage.k8s.io",
version="v1",
namespace=test_namespace,
plural="volumesnapshots",
body={
"apiVersion": "snapshot.storage.k8s.io/v1",
"kind": "VolumeSnapshot",
"metadata": {"name": snap, "namespace": test_namespace},
"spec": {
"volumeSnapshotClassName": cluster_config.kql_snapshot_class(),
"source": {"persistentVolumeClaimName": f"{source}-root-lh"},
},
},
)
# Wait for ready.
deadline = time.time() + 180
ready = False
while time.time() < deadline:
info = await k7.get_snapshot(snap, namespace=test_namespace)
if info is not None and info.ready_to_use:
ready = True
break
await asyncio.sleep(2)
assert ready, "legacy snapshot never became ready"
# Restore *without* --image must fail loudly.
restore_k7 = K7Core()
bad = await restore_k7.restore_sandbox(
snapshot_name=snap,
new_sandbox_name=restored,
namespace=test_namespace,
)
assert not bad.success
assert "k7.io/source-image" in bad.error
assert "--image" in bad.error
# Restore *with* --image succeeds.
good = await restore_k7.restore_sandbox(
snapshot_name=snap,
new_sandbox_name=restored,
namespace=test_namespace,
overrides=SandboxConfigOverrides(image="alpine:3.20"),
)
assert good.success, good.error
_wait_pod_ready(restored, test_namespace)
class TestRestoreNoKeepSnapshot:
async def test_no_keep_snapshot_deletes_snapshot(
self,
test_namespace: str,
restore_sandbox_cleanup,
):
register_sandbox, register_snapshot = restore_sandbox_cleanup
source = f"restore-nk-{uuid4().hex[:6]}"
snap = f"{source}-disposable"
restored = f"{source}-r"
register_sandbox(source)
register_sandbox(restored)
register_snapshot(snap)
k7 = K7Core()
await k7.create_sandbox(_ql_config(source, test_namespace))
_wait_pod_ready(source, test_namespace)
await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
# Wait for ready.
deadline = time.time() + 180
while time.time() < deadline:
info = await k7.get_snapshot(snap, namespace=test_namespace)
if info is not None and info.ready_to_use:
break
await asyncio.sleep(2)
restore_k7 = K7Core()
result = await restore_k7.restore_sandbox(
snapshot_name=snap,
new_sandbox_name=restored,
namespace=test_namespace,
keep_snapshot=False,
)
assert result.success, result.error
_wait_pod_ready(restored, test_namespace)
# Snapshot should be deleted (give Longhorn a few seconds to settle).
deadline = time.time() + 30
while time.time() < deadline and _snapshot_exists(snap, test_namespace):
time.sleep(1)
assert not _snapshot_exists(snap, test_namespace)
class TestRestoreErrors:
async def test_restore_into_existing_name_fails(
self,
test_namespace: str,
restore_sandbox_cleanup,
):
register_sandbox, register_snapshot = restore_sandbox_cleanup
source = f"restore-conf-{uuid4().hex[:6]}"
snap = f"{source}-snap"
register_sandbox(source)
register_snapshot(snap)
k7 = K7Core()
await k7.create_sandbox(_ql_config(source, test_namespace))
_wait_pod_ready(source, test_namespace)
await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
deadline = time.time() + 180
while time.time() < deadline:
info = await k7.get_snapshot(snap, namespace=test_namespace)
if info is not None and info.ready_to_use:
break
await asyncio.sleep(2)
# Restore into the source's *own* name → must fail.
restore_k7 = K7Core()
result = await restore_k7.restore_sandbox(
snapshot_name=snap,
new_sandbox_name=source, # already exists
namespace=test_namespace,
)
assert not result.success
assert "already exists" in result.error
async def test_restore_missing_snapshot_returns_helpful_error(self, test_namespace: str):
k7 = K7Core()
result = await k7.restore_sandbox(
snapshot_name="does-not-exist",
new_sandbox_name="any",
namespace=test_namespace,
)
assert not result.success
assert "not found" in result.error.lower()