chore: record terminal operator user testing rerun

This commit is contained in:
RCD
2026-04-05 20:41:25 +07:00
parent 53be957861
commit a7ea27d047
13 changed files with 411 additions and 58 deletions
@@ -0,0 +1,19 @@
active_account = "refreshable-local"
[[accounts]]
name = "refreshable-local"
provider = "codex"
api_key = "refreshed-access-token"
refresh_token = "refreshed-refresh-token"
id_token = "header.eyJlbWFpbCI6InJlcnVuQGV4YW1wbGUuY29tIiwiZXhwIjoxODkzNDU2MDAwLCJodHRwczovL2FwaS5vcGVuYWkuY29tL2F1dGgiOnsiY2hhdGdwdF9hY2NvdW50X2lkIjoiYWNjdF9yZWZyZXNoX3JlcnVuIiwiY2hhdGdwdF9wbGFuX3R5cGUiOiJwcm8ifX0.sig"
email = "rerun@example.com"
account_id = "acct_refresh_rerun"
plan_type = "pro"
expires_at = "2030-01-01T00:00:00Z"
source = "import:/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/cli-refresh-rerun/refreshable-auth.json"
[[accounts]]
name = "static-local"
provider = "codex"
api_key = "sk-static-local"
source = "manual"
@@ -0,0 +1,11 @@
# isolated CLI refresh rerun config
[server]
bind = "127.0.0.1:8318"
[state]
path = "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/cli-refresh-rerun/proxypilot-rs.state.toml"
[codex]
upstream_base_url = "https://api.openai.com"
api_key = ""
refresh_token_url = "http://127.0.0.1:34051/oauth/token"
@@ -0,0 +1,8 @@
{
"access_token": "seed-access-token",
"refresh_token": "seed-refresh-token",
"email": "seed@example.com",
"account_id": "acct_seed",
"expired": "2020-01-01T00:00:00Z",
"type": "codex"
}
@@ -0,0 +1,13 @@
active_account = "preemptive-account"
[[accounts]]
name = "preemptive-account"
provider = "codex"
api_key = "expired-preemptive-token"
refresh_token = "preemptive-refresh-token"
id_token = "header.payload.sig"
email = "refresh@example.com"
account_id = "acct_seed"
plan_type = "seed"
expires_at = "1970-01-01T00:00:00Z"
source = "fixture"
@@ -0,0 +1,10 @@
[server]
bind = "127.0.0.1:18318"
[state]
path = "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/http-refresh-rerun/VAL-HTTP-007.state.toml"
[codex]
upstream_base_url = "http://127.0.0.1:18319"
api_key = ""
refresh_token_url = "http://127.0.0.1:18319/oauth/token"
@@ -0,0 +1,13 @@
active_account = "retry-account"
[[accounts]]
name = "retry-account"
provider = "codex"
api_key = "stale-401-token"
refresh_token = "retry-refresh-token"
id_token = "header.payload.sig"
email = "refresh@example.com"
account_id = "acct_seed"
plan_type = "seed"
expires_at = "2035-01-01T00:00:00Z"
source = "fixture"
@@ -0,0 +1,10 @@
[server]
bind = "127.0.0.1:18318"
[state]
path = "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/http-refresh-rerun/VAL-HTTP-008.state.toml"
[codex]
upstream_base_url = "http://127.0.0.1:18319"
api_key = ""
refresh_token_url = "http://127.0.0.1:18319/oauth/token"
@@ -0,0 +1,142 @@
import argparse
import base64
import json
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs
def encode_claims(exp, email, account_id, plan_type):
payload = {
"email": email,
"exp": exp,
"https://api.openai.com/auth": {
"chatgpt_account_id": account_id,
"chatgpt_plan_type": plan_type,
},
}
raw = json.dumps(payload, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
class ReusableServer(ThreadingHTTPServer):
allow_reuse_address = True
class Handler(BaseHTTPRequestHandler):
scenario = "preemptive"
log_path = None
state = {"model_calls": 0}
def log_message(self, format, *args):
return
def _write_json(self, status, payload):
body = json.dumps(payload).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return body
def _record(self, *, response_status, response_body, request_body=""):
entry = {
"ts": time.time(),
"scenario": self.scenario,
"method": self.command,
"path": self.path,
"headers": {k: v for k, v in self.headers.items()},
"request_body": request_body,
"response_status": response_status,
"response_body": response_body,
"model_call_index": self.state["model_calls"],
}
with open(self.log_path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(entry) + "\n")
def do_GET(self):
if self.path == "/__health":
body = b"ok"
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if self.path == "/v1/models":
auth = self.headers.get("Authorization", "")
self.state["model_calls"] += 1
if self.scenario == "retry401" and self.state["model_calls"] == 1:
payload = {
"error": "expired token",
"received_authorization": auth,
"attempt": 1,
}
self._record(response_status=401, response_body=payload)
self._write_json(401, payload)
return
payload = {
"object": "list",
"data": [{"id": "gpt-5.2-codex", "object": "model"}],
"received_authorization": auth,
"attempt": self.state["model_calls"],
"scenario": self.scenario,
}
self._record(response_status=200, response_body=payload)
self._write_json(200, payload)
return
payload = {"error": "not found", "path": self.path}
self._record(response_status=404, response_body=payload)
self._write_json(404, payload)
def do_POST(self):
length = int(self.headers.get("Content-Length", "0"))
raw_body = self.rfile.read(length).decode()
if self.path == "/oauth/token":
form = parse_qs(raw_body)
refresh_token = form.get("refresh_token", [""])[0]
if self.scenario == "preemptive":
payload = {
"access_token": "preemptive-fresh-token",
"refresh_token": "preemptive-fresh-refresh-token",
"id_token": f"header.{encode_claims(1893456000, 'refresh@example.com', 'acct_preemptive', 'team')}.sig",
"expires_in": 3600,
"seen_refresh_token": refresh_token,
}
else:
payload = {
"access_token": "retry-fresh-token",
"refresh_token": "retry-fresh-refresh-token",
"id_token": f"header.{encode_claims(1893456000, 'refresh@example.com', 'acct_retry401', 'pro')}.sig",
"expires_in": 3600,
"seen_refresh_token": refresh_token,
}
self._record(response_status=200, response_body=payload, request_body=raw_body)
self._write_json(200, payload)
return
payload = {"error": "not found", "path": self.path}
self._record(response_status=404, response_body=payload, request_body=raw_body)
self._write_json(404, payload)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--scenario", required=True, choices=["preemptive", "retry401"])
parser.add_argument("--port", required=True, type=int)
parser.add_argument("--log", required=True)
args = parser.parse_args()
Handler.scenario = args.scenario
Handler.log_path = args.log
Handler.state = {"model_calls": 0}
server = ReusableServer(("127.0.0.1", args.port), Handler)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
finally:
server.server_close()
@@ -0,0 +1,69 @@
{
"groupId": "cli-refresh-rerun",
"testedAt": "2026-04-05T13:32:26Z",
"isolation": {
"surface": "shell",
"fixtureDirectory": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/cli-refresh-rerun",
"evidenceDirectory": "/home/fsos/.factory/missions/7e6968f8-b3d3-4772-9518-ca3a93ea1374/evidence/terminal-operator-console/cli-refresh-rerun",
"configPath": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/cli-refresh-rerun/proxypilot-rs.toml",
"statePath": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/cli-refresh-rerun/proxypilot-rs.state.toml",
"refreshTokenUrl": "http://127.0.0.1:34051/oauth/token"
},
"toolsUsed": [
"shell"
],
"assertions": [
{
"id": "VAL-CLI-003",
"title": "Refresh commands handle both refreshable and static accounts clearly",
"status": "pass",
"steps": [
{
"action": "Import a refreshable saved Codex account into isolated state and add a static account",
"expected": "Fixture state contains one active refreshable account and one static account",
"observed": "account list showed `refreshable-local` active with `refresh=refreshable` and `static-local` present with `refresh=static`"
},
{
"action": "Run `account refresh-codex` with no `--name` while the refreshable account is active",
"expected": "Refresh succeeds through the configured local refresh endpoint and updates saved account metadata",
"observed": "Command printed `refreshed saved Codex account `refreshable-local` in local state`; the local stub received one POST to `/oauth/token`; state updated access token, refresh token, email, account id, plan, and expiry"
},
{
"action": "Run `account refresh-codex --name static-local`",
"expected": "Static account refresh is rejected with a clear operator-readable error and no state corruption",
"observed": "Command exited non-zero with `account `static-local` is static and has no refresh token`; follow-up state snapshot matched the post-success state byte-for-byte"
},
{
"action": "Run `account refresh-codex --name missing-local`",
"expected": "Missing account refresh is rejected with a clear operator-readable error and no state corruption",
"observed": "Command exited non-zero with `no saved Codex account named missing-local`; follow-up state snapshot remained unchanged and list output still showed the same active/static accounts"
}
],
"evidence": {
"commandOutputs": [
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-import-refreshable.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-add-static.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-list-before-refresh.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-refresh-success.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-list-after-success.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-refresh-static-fails.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-refresh-missing-fails.txt",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-list-after-failures.txt"
],
"stateSnapshots": [
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-state-before-refresh.toml",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-state-after-success.toml",
"terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-state-after-failures.toml"
],
"network": "terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-refresh-stub-requests.json",
"config": "terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-config.toml",
"fixtureImport": "terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-import-auth.json",
"validator": "terminal-operator-console/cli-refresh-rerun/VAL-CLI-003-refresh-related-tests.txt"
},
"issues": null
}
],
"frictions": [],
"blockers": [],
"summary": "VAL-CLI-003 passed: refresh succeeded for the active refreshable Codex account through `codex.refresh_token_url`, static and missing account refreshes failed clearly, and saved state stayed coherent after all runs."
}
@@ -0,0 +1,84 @@
{
"groupId": "http-refresh-rerun",
"testedAt": "2026-04-05T13:34:49Z",
"isolation": {
"surface": "curl",
"fixtureDirectory": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/fixtures/http-refresh-rerun",
"evidenceDirectory": "/home/fsos/.factory/missions/7e6968f8-b3d3-4772-9518-ca3a93ea1374/evidence/terminal-operator-console/http-refresh-rerun",
"proxyBind": "127.0.0.1:18318",
"stubBind": "127.0.0.1:18319",
"refreshTokenUrl": "http://127.0.0.1:18319/oauth/token"
},
"toolsUsed": [
"curl"
],
"assertions": [
{
"id": "VAL-HTTP-007",
"title": "Proactive refresh is visible before forwarding expired or near-expiry requests",
"status": "pass",
"steps": [
{
"action": "Start proxypilot-rs with an expired refreshable active account plus a local OAuth/upstream stub",
"expected": "Runtime stats start with expired auth health and zero counters",
"observed": "/v0/runtime/stats reported auth_health.state=expired with total_proxied_requests=0 and auth_refresh_attempts=0"
},
{
"action": "Send GET /v1/models through the proxy",
"expected": "Proxy refreshes before forwarding and the client request succeeds with the refreshed token",
"observed": "Client received 200 JSON showing received_authorization=Bearer preemptive-fresh-token"
},
{
"action": "Inspect runtime stats and stub traffic after the request",
"expected": "Stats show one refresh attempt/success and upstream sees only refreshed-token traffic",
"observed": "Stats showed total_proxied_requests=1, successful_upstream_responses=1, auth_refresh_attempts=1, upstream_401_count=0, last_refresh.kind=success; stub log recorded POST /oauth/token before GET /v1/models with Bearer preemptive-fresh-token"
}
],
"evidence": {
"statsBefore": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-stats-before.json",
"requestHeaders": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-request.headers.txt",
"requestBody": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-request-body.json",
"statsAfter": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-stats-after.json",
"stubLog": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-stub-log.jsonl",
"proxyLog": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-proxy.log",
"stubStdout": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-007-stub.log"
},
"issues": null
},
{
"id": "VAL-HTTP-008",
"title": "Upstream 401 retry refresh is visible and counted exactly once",
"status": "pass",
"steps": [
{
"action": "Start proxypilot-rs with a refreshable active account and a local stub that returns 401 on the first /v1/models request",
"expected": "Initial stats are zeroed and the first upstream attempt uses the stale token",
"observed": "Stub log recorded the first GET /v1/models with Bearer stale-401-token and response_status=401"
},
{
"action": "Let the proxy handle the 401-triggered refresh-and-retry path for one GET /v1/models client request",
"expected": "Proxy refreshes once, retries once, and the client sees the retried success result",
"observed": "Client received 200 JSON showing received_authorization=Bearer retry-fresh-token on attempt=2"
},
{
"action": "Inspect runtime stats and stub traffic after the request",
"expected": "Stats count one proxied request, one upstream 401, one refresh success, and mock upstream shows 401 then refresh then retry",
"observed": "Stats showed total_proxied_requests=1, successful_upstream_responses=1, auth_refresh_attempts=1, upstream_401_count=1, last_refresh.kind=success; stub log showed GET 401 -> POST /oauth/token -> GET 200 with Bearer retry-fresh-token"
}
],
"evidence": {
"statsBefore": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-stats-before.json",
"requestHeaders": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-request.headers.txt",
"requestBody": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-request-body.json",
"statsAfter": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-stats-after.json",
"stubLog": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-stub-log.jsonl",
"proxyLog": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-proxy.log",
"stubStdout": "terminal-operator-console/http-refresh-rerun/VAL-HTTP-008-stub.log"
},
"issues": null
}
],
"frictions": [],
"blockers": [],
"summary": "Retested 2 assertions: VAL-HTTP-007 passed with proactive refresh before forwarding, and VAL-HTTP-008 passed with one 401-triggered refresh-and-retry path counted exactly once."
}
@@ -1,66 +1,20 @@
{
"milestone": "terminal-operator-console",
"round": 1,
"status": "fail",
"round": 2,
"status": "pass",
"assertionsSummary": {
"total": 28,
"passed": 25,
"total": 3,
"passed": 3,
"failed": 0,
"blocked": 3
"blocked": 0
},
"passedAssertions": [
"VAL-CLI-001",
"VAL-CLI-002",
"VAL-HTTP-001",
"VAL-HTTP-002",
"VAL-HTTP-003",
"VAL-HTTP-004",
"VAL-HTTP-005",
"VAL-HTTP-006",
"VAL-HTTP-009",
"VAL-HTTP-010",
"VAL-TUI-001",
"VAL-TUI-002",
"VAL-TUI-003",
"VAL-TUI-004",
"VAL-TUI-005",
"VAL-TUI-006",
"VAL-TUI-007",
"VAL-TUI-008",
"VAL-TUI-009",
"VAL-TUI-010",
"VAL-TUI-011",
"VAL-TUI-012",
"VAL-CROSS-001",
"VAL-CROSS-002",
"VAL-CROSS-003"
"VAL-CLI-003",
"VAL-HTTP-007",
"VAL-HTTP-008"
],
"failedAssertions": [],
"blockedAssertions": [
{
"id": "VAL-CLI-003",
"blockedBy": "Negative refresh paths behaved clearly for static and missing accounts, but the required refresh-success path could not be exercised within the isolated fixture because refresh-codex is hardwired to the live OpenAI OAuth refresh endpoint."
},
{
"id": "VAL-HTTP-007",
"blockedBy": "A proactive refresh attempt was observable before forwarding, but the success path could not be validated because the runtime cannot be redirected from the user surface to a local OAuth token stub."
},
{
"id": "VAL-HTTP-008",
"blockedBy": "The 401-triggered refresh path is visible, but the success retry case cannot be validated locally because the runtime cannot point refresh traffic at a local OAuth token stub from the user surface."
}
],
"appliedUpdates": [
{
"target": "user-testing.md",
"description": "Added flow validator guidance for shell, curl, and tuistory surfaces plus isolated fixture/evidence boundaries for terminal-operator-console validation.",
"source": "setup"
},
{
"target": "user-testing.md",
"description": "Documented discovered validation quirks: hardwired OAuth refresh paths block local success-path refresh testing, tuistory PTYs need explicit close, and runtime-down selected refresh still surfaces remote OAuth failure text.",
"source": "flow-report"
}
],
"previousRound": null
"blockedAssertions": [],
"appliedUpdates": [],
"previousRound": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/synthesis.round1.json"
}
@@ -0,0 +1,20 @@
{
"milestone": "terminal-operator-console",
"round": 2,
"status": "pass",
"assertionsSummary": {
"total": 3,
"passed": 3,
"failed": 0,
"blocked": 0
},
"passedAssertions": [
"VAL-CLI-003",
"VAL-HTTP-007",
"VAL-HTTP-008"
],
"failedAssertions": [],
"blockedAssertions": [],
"appliedUpdates": [],
"previousRound": "/home/fsos/code-lean/ai-tools/ProxyPilot/.factory/validation/terminal-operator-console/user-testing/synthesis.round1.json"
}
+1 -1
View File
@@ -454,7 +454,7 @@ mod tests {
Some("2030-01-01T00:00:00Z")
);
let _ = server.abort();
server.abort();
let _ = fs::remove_file(config_path);
let _ = fs::remove_file(state_path);
}