mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-20 20:34:19 +00:00
* feat(mcp): pin tool definitions in mcp check --live and warn when they change An Agent reads a server's tool names and descriptions as instructions, and a server can change them after the user reviewed it (a "rug pull"). The MCP spec 2026-07-28 local-server security guide notes the protocol does nothing about this and advises clients that re-prompt on a changed definition; most clients do not. `mcp check --live` now records one SHA-256 per tool over name, description and inputSchema the first time a server answers, in mcp/tool-pins.json of the state directory, keyed by scope and server. Later probes report a changed or new tool as a `tool-change` warning and a removed one as info, until `--accept-tools` (only with --live) records the current definitions. Without --live the pins are never read or written. An unreadable pin file is never rewritten, since it may hold other servers' pins. The dashboard words both warnings in every locale. * fix(mcp): keep schema numbers exact, name the scope to accept in, and log acceptance Codex review on #560: - Tool definitions were decoded through float64 before hashing, so a schema bound changed from 9007199254740992 to 9007199254740993 hashed the same and raised no warning. Numbers now keep their literal text. - The suggested accept command left out -g/-p; run from inside a project after a global check, it would accept the project's server instead. It now names the scope of the check that warned. - Accepting changed tool definitions approves a possible rug pull, but left no record. --accept-tools now writes an operations log entry with the servers it accepted. * fix(mcp): log a tool acceptance only when its baseline was saved Codex review on #560: when comparing succeeded but writing tool-pins.json failed (lock held, state directory unwritable), the report kept its "accepted" finding, so the operations log listed the server as accepted with status ok although its old pins remained. Such a server is now logged under `failed` and the entry's status is error. * fix(mcp): log first pins as accepted and report errors in the accept entry Codex review on #560: - A check that accepted one server but found an error on another still logged status ok, while the command itself exited 1. A server with an error is now listed under `failed` and the entry's status is error. - --accept-tools on a server's first probe records its baseline, but the finding says "pinned", so the entry listed nothing as accepted. A first pin counts as accepted now. * chore(release): request version 0.25.11 The MCP pin checks (#559, #560) and the backup staging fix (#558) ship as 0.25.11 instead of the computed 0.26.0, per maintainer decision. Release-As: 0.25.11
378 lines
15 KiB
Go
378 lines
15 KiB
Go
package mcp
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const liveSecret = "s3cr3t-live-value"
|
|
|
|
// TestMCPLiveHelperServer is not a test: the live probe tests start this test binary as an
|
|
// MCP server, selected by MCP_LIVE_HELPER, speaking the mode given after --.
|
|
func TestMCPLiveHelperServer(t *testing.T) {
|
|
if os.Getenv("MCP_LIVE_HELPER") != "1" {
|
|
return
|
|
}
|
|
mode := os.Args[len(os.Args)-1]
|
|
if marker := os.Getenv("MCP_LIVE_MARKER"); marker != "" {
|
|
_ = os.WriteFile(marker, []byte("started"), 0600)
|
|
}
|
|
if os.Getenv("API_TOKEN") != liveSecret {
|
|
fmt.Fprintln(os.Stderr, "API_TOKEN was not passed")
|
|
os.Exit(4)
|
|
}
|
|
switch mode {
|
|
case "crash":
|
|
fmt.Fprintf(os.Stderr, "starting\nfatal: token %s rejected\n", os.Getenv("API_TOKEN"))
|
|
os.Exit(3)
|
|
case "hang":
|
|
signal.Ignore(syscall.SIGTERM)
|
|
if os.Getenv("MCP_LIVE_CHILD") == "" {
|
|
child := exec.Command(os.Args[0], os.Args[1:]...)
|
|
child.Env = append(os.Environ(), "MCP_LIVE_CHILD=1")
|
|
if err := child.Start(); err != nil {
|
|
os.Exit(5)
|
|
}
|
|
_ = os.WriteFile(os.Getenv("MCP_LIVE_PIDS"), fmt.Appendf(nil, "%d %d", os.Getpid(), child.Process.Pid), 0600)
|
|
}
|
|
select {}
|
|
}
|
|
out := json.NewEncoder(os.Stdout)
|
|
reply := func(id json.RawMessage, result any) {
|
|
_ = out.Encode(map[string]any{"jsonrpc": "2.0", "id": id, "result": result})
|
|
}
|
|
initialized := false
|
|
scanner := bufio.NewScanner(os.Stdin)
|
|
for scanner.Scan() {
|
|
var msg struct {
|
|
ID json.RawMessage `json:"id"`
|
|
Method string `json:"method"`
|
|
Params struct {
|
|
Cursor string `json:"cursor"`
|
|
} `json:"params"`
|
|
}
|
|
if json.Unmarshal(scanner.Bytes(), &msg) != nil {
|
|
os.Exit(6)
|
|
}
|
|
switch {
|
|
case mode == "pinned" && msg.Method == "tools/list":
|
|
var tools []map[string]any
|
|
_ = json.Unmarshal([]byte(os.Getenv("MCP_LIVE_TOOLS")), &tools)
|
|
reply(msg.ID, map[string]any{"resultType": "complete", "tools": tools})
|
|
case (mode == "modern" || mode == "pinned") && msg.Method == "server/discover":
|
|
reply(msg.ID, map[string]any{"resultType": "complete", "supportedVersions": []string{"2026-07-28"}, "capabilities": map[string]any{"tools": map[string]any{}},
|
|
"_meta": map[string]any{"io.modelcontextprotocol/serverInfo": map[string]string{"name": "helper", "version": "1.2.3"}}})
|
|
case mode == "modern" && msg.Method == "tools/list" && msg.Params.Cursor == "":
|
|
reply(msg.ID, map[string]any{"resultType": "complete", "tools": []map[string]string{{"name": "a"}}, "nextCursor": "page2"})
|
|
case mode == "modern" && msg.Method == "tools/list":
|
|
reply(msg.ID, map[string]any{"resultType": "complete", "tools": []map[string]string{{"name": "b"}}})
|
|
case msg.Method == "initialize":
|
|
reply(msg.ID, map[string]any{"protocolVersion": "2025-11-25", "capabilities": map[string]any{"tools": map[string]any{}}, "serverInfo": map[string]string{"name": "legacy", "version": "0.9"}})
|
|
case msg.Method == "notifications/initialized":
|
|
initialized = true
|
|
case msg.Method == "tools/list" && initialized:
|
|
reply(msg.ID, map[string]any{"tools": []map[string]string{{"name": "a"}, {"name": "b"}, {"name": "c"}}})
|
|
case mode == "silent":
|
|
// Some legacy servers never answer a request they do not know.
|
|
default:
|
|
_ = out.Encode(map[string]any{"jsonrpc": "2.0", "id": msg.ID, "error": map[string]any{"code": -32601, "message": "Method not found"}})
|
|
}
|
|
}
|
|
os.Exit(0)
|
|
}
|
|
|
|
// liveConfig declares one stdio server that runs the helper above in mode.
|
|
func liveConfig(mode string, env map[string]string) string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "mcp:\n targets: [claude]\n servers:\n fake:\n command: %q\n args: [%q, \"--\", %q]\n env:\n MCP_LIVE_HELPER: \"1\"\n API_TOKEN: {fromEnv: LIVE_TOKEN}\n", os.Args[0], "-test.run=^TestMCPLiveHelperServer$", mode)
|
|
for _, key := range sortedKeys(env) {
|
|
fmt.Fprintf(&b, " %s: %q\n", key, env[key])
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func liveCheck(timeout time.Duration, env map[string]string) CheckOptions {
|
|
opts := offlineCheck()
|
|
opts.Live, opts.Timeout = true, timeout
|
|
opts.LookPath = exec.LookPath
|
|
opts.LookupEnv = func(name string) (string, bool) {
|
|
value, ok := env[name]
|
|
return value, ok
|
|
}
|
|
return opts
|
|
}
|
|
|
|
func liveOf(t *testing.T, report *CheckReport) (*CheckLive, CheckFinding) {
|
|
t.Helper()
|
|
for _, f := range report.Servers[0].Findings {
|
|
if f.Check == "live" {
|
|
return report.Servers[0].Live, f
|
|
}
|
|
}
|
|
t.Fatalf("no live finding: %+v", report.Servers[0].Findings)
|
|
return nil, CheckFinding{}
|
|
}
|
|
|
|
func assertNoSecret(t *testing.T, report *CheckReport) {
|
|
t.Helper()
|
|
data, _ := json.Marshal(report)
|
|
if strings.Contains(string(data), liveSecret) {
|
|
t.Fatalf("report leaks a configured value: %s", data)
|
|
}
|
|
}
|
|
|
|
var tokenEnv = map[string]string{"LIVE_TOKEN": liveSecret}
|
|
|
|
func TestLiveStdioDiscover(t *testing.T) {
|
|
report, err := checkService(t, liveConfig("modern", nil)).Check(liveCheck(10*time.Second, tokenEnv))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
live, f := liveOf(t, report)
|
|
want := CheckLive{ProtocolVersion: "2026-07-28", ServerInfo: LiveServerInfo{Name: "helper", Version: "1.2.3"}, Tools: 2, ToolNames: []string{"a", "b"}}
|
|
if f.Level != "info" || live == nil || !reflect.DeepEqual(*live, want) || !report.Servers[0].OK {
|
|
t.Fatalf("live = %+v, finding = %+v", live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveStdioFallsBackToInitialize(t *testing.T) {
|
|
report, err := checkService(t, liveConfig("legacy", nil)).Check(liveCheck(10*time.Second, tokenEnv))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
live, f := liveOf(t, report)
|
|
want := CheckLive{ProtocolVersion: "2025-11-25", ServerInfo: LiveServerInfo{Name: "legacy", Version: "0.9"}, Tools: 3, ToolNames: []string{"a", "b", "c"}}
|
|
if live == nil || !reflect.DeepEqual(*live, want) {
|
|
t.Fatalf("live = %+v, finding = %+v", live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveStdioFallsBackWhenDiscoverIsUnanswered(t *testing.T) {
|
|
report, err := checkService(t, liveConfig("silent", nil)).Check(liveCheck(3*time.Second, tokenEnv))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if live, f := liveOf(t, report); live == nil || live.ProtocolVersion != "2025-11-25" {
|
|
t.Fatalf("live = %+v, finding = %+v", live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveStdioExitReportsRedactedStderr(t *testing.T) {
|
|
report, err := checkService(t, liveConfig("crash", nil)).Check(liveCheck(10*time.Second, tokenEnv))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, f := liveOf(t, report)
|
|
if f.Level != "error" || !strings.Contains(f.Message, "exit status 3") || !strings.Contains(f.Message, "token [redacted] rejected") || report.Summary.Errors != 1 {
|
|
t.Fatalf("finding = %+v", f)
|
|
}
|
|
assertNoSecret(t, report)
|
|
}
|
|
|
|
func TestLiveStdioSpawnFailure(t *testing.T) {
|
|
s := checkService(t, "mcp:\n targets: [claude]\n servers:\n gone:\n command: /no/such/mcp-server\n")
|
|
opts := liveCheck(time.Second, nil)
|
|
// The static check passes; starting the command is what fails.
|
|
opts.LookPath = func(name string) (string, error) { return name, nil }
|
|
report, err := s.Check(opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, f := liveOf(t, report); f.Level != "error" || !strings.Contains(f.Message, "could not start /no/such/mcp-server") {
|
|
t.Fatalf("finding = %+v", f)
|
|
}
|
|
}
|
|
|
|
func TestLiveAbsentStartsNothing(t *testing.T) {
|
|
marker := filepath.Join(t.TempDir(), "started")
|
|
opts := liveCheck(time.Second, tokenEnv)
|
|
opts.Live = false
|
|
report, err := checkService(t, liveConfig("modern", map[string]string{"MCP_LIVE_MARKER": marker})).Check(opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(marker); !os.IsNotExist(err) {
|
|
t.Fatal("a check without Live started the server")
|
|
}
|
|
if report.Servers[0].Live != nil || subjectOf(report.Servers[0].Findings, "live") != "" || hasFinding(report.Servers[0].Findings, "info", "live", "") {
|
|
t.Fatalf("unexpected live result: %+v", report.Servers[0])
|
|
}
|
|
}
|
|
|
|
func TestLiveSkipsServerWithStaticError(t *testing.T) {
|
|
marker := filepath.Join(t.TempDir(), "started")
|
|
report, err := checkService(t, liveConfig("modern", map[string]string{"MCP_LIVE_MARKER": marker})).Check(liveCheck(time.Second, nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, f := liveOf(t, report); f.Level != "info" || !strings.Contains(f.Message, "not probed live") {
|
|
t.Fatalf("finding = %+v", f)
|
|
}
|
|
if _, err := os.Stat(marker); !os.IsNotExist(err) {
|
|
t.Fatal("a server with a missing variable was started")
|
|
}
|
|
}
|
|
|
|
// liveHTTP serves a check config for one remote server backed by handler.
|
|
func liveHTTP(t *testing.T, handler http.HandlerFunc) (*CheckReport, CheckFinding) {
|
|
t.Helper()
|
|
ts := httptest.NewServer(handler)
|
|
defer ts.Close()
|
|
s := checkService(t, fmt.Sprintf("mcp:\n targets: [claude]\n servers:\n remote:\n url: %s/mcp\n bearerToken: {fromEnv: LIVE_TOKEN}\n headers:\n X-Team: {fromEnv: LIVE_TEAM}\n", ts.URL))
|
|
report, err := s.Check(liveCheck(5*time.Second, map[string]string{"LIVE_TOKEN": liveSecret, "LIVE_TEAM": "team-" + liveSecret}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertNoSecret(t, report)
|
|
_, f := liveOf(t, report)
|
|
return report, f
|
|
}
|
|
|
|
type httpRequest struct {
|
|
ID json.RawMessage `json:"id"`
|
|
Method string `json:"method"`
|
|
}
|
|
|
|
func decodeRequest(t *testing.T, r *http.Request) httpRequest {
|
|
var req httpRequest
|
|
body, _ := io.ReadAll(r.Body)
|
|
if err := json.Unmarshal(body, &req); err != nil {
|
|
t.Errorf("invalid request body %s", body)
|
|
}
|
|
if r.Header.Get("Authorization") != "Bearer "+liveSecret || r.Header.Get("X-Team") != "team-"+liveSecret || r.Header.Get("Mcp-Method") != req.Method {
|
|
t.Errorf("%s: missing or wrong headers: %v", req.Method, r.Header)
|
|
}
|
|
return req
|
|
}
|
|
|
|
func TestLiveHTTPDiscoverJSON(t *testing.T) {
|
|
report, f := liveHTTP(t, func(w http.ResponseWriter, r *http.Request) {
|
|
req := decodeRequest(t, r)
|
|
if r.Header.Get("MCP-Protocol-Version") != "2026-07-28" {
|
|
t.Errorf("missing protocol version header")
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
result := map[string]any{"resultType": "complete", "tools": []any{map[string]string{"name": "x"}}}
|
|
if req.Method == "server/discover" {
|
|
result = map[string]any{"resultType": "complete", "supportedVersions": []string{"2026-07-28"}, "capabilities": map[string]any{"tools": map[string]any{}},
|
|
"_meta": map[string]any{"io.modelcontextprotocol/serverInfo": map[string]string{"name": "remote", "version": "2"}}}
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result})
|
|
})
|
|
want := CheckLive{ProtocolVersion: "2026-07-28", ServerInfo: LiveServerInfo{Name: "remote", Version: "2"}, Tools: 1, ToolNames: []string{"x"}}
|
|
if f.Level != "info" || report.Servers[0].Live == nil || !reflect.DeepEqual(*report.Servers[0].Live, want) {
|
|
t.Fatalf("live = %+v, finding = %+v", report.Servers[0].Live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveHTTPSSEResponse(t *testing.T) {
|
|
report, f := liveHTTP(t, func(w http.ResponseWriter, r *http.Request) {
|
|
req := decodeRequest(t, r)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, ": keep-alive\n\nevent: message\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/progress\",\"params\":{}}\n\n")
|
|
result, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": map[string]any{"supportedVersions": []string{"2026-07-28"}, "capabilities": map[string]any{},
|
|
"_meta": map[string]any{"io.modelcontextprotocol/serverInfo": map[string]string{"name": "sse", "version": "1"}}}})
|
|
fmt.Fprintf(w, "event: message\ndata: %s\n\n", result)
|
|
})
|
|
if f.Level != "info" || report.Servers[0].Live == nil || report.Servers[0].Live.ServerInfo.Name != "sse" || report.Servers[0].Live.Tools != 0 {
|
|
t.Fatalf("live = %+v, finding = %+v", report.Servers[0].Live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveHTTPFallsBackToInitialize(t *testing.T) {
|
|
report, f := liveHTTP(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodDelete {
|
|
return
|
|
}
|
|
req := decodeRequest(t, r)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch {
|
|
case req.Method == "initialize":
|
|
w.Header().Set("Mcp-Session-Id", "session-1")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": map[string]any{"protocolVersion": "2025-06-18", "capabilities": map[string]any{"tools": map[string]any{}}, "serverInfo": map[string]string{"name": "old", "version": "1"}}})
|
|
case r.Header.Get("Mcp-Session-Id") != "session-1":
|
|
http.Error(w, "Bad Request: No valid session ID provided", http.StatusBadRequest)
|
|
case req.Method == "notifications/initialized":
|
|
w.WriteHeader(http.StatusAccepted)
|
|
case r.Header.Get("MCP-Protocol-Version") != "2025-06-18":
|
|
http.Error(w, "wrong protocol version", http.StatusBadRequest)
|
|
default:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": map[string]any{"tools": []any{map[string]string{"name": "x"}, map[string]string{"name": "y"}}}})
|
|
}
|
|
})
|
|
want := CheckLive{ProtocolVersion: "2025-06-18", ServerInfo: LiveServerInfo{Name: "old", Version: "1"}, Tools: 2, ToolNames: []string{"x", "y"}}
|
|
if report.Servers[0].Live == nil || !reflect.DeepEqual(*report.Servers[0].Live, want) {
|
|
t.Fatalf("live = %+v, finding = %+v", report.Servers[0].Live, f)
|
|
}
|
|
}
|
|
|
|
func TestLiveHTTPUnauthorizedIsWarning(t *testing.T) {
|
|
calls := 0
|
|
report, f := liveHTTP(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
w.Header().Set("WWW-Authenticate", `Bearer resource_metadata="https://auth.example.com/.well-known/oauth-protected-resource", scope="files:read"`)
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
})
|
|
if f.Level != "warning" || f.Subject != "https://auth.example.com/.well-known/oauth-protected-resource" || !strings.Contains(f.Message, "sign-in required") || !report.Servers[0].OK || calls != 1 {
|
|
t.Fatalf("finding = %+v, calls = %d", f, calls)
|
|
}
|
|
}
|
|
|
|
func TestLiveHTTPServerErrorIsError(t *testing.T) {
|
|
report, f := liveHTTP(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "boom "+r.Header.Get("Authorization"), http.StatusInternalServerError)
|
|
})
|
|
if f.Level != "error" || !strings.Contains(f.Message, "HTTP 500") || report.Servers[0].OK {
|
|
t.Fatalf("finding = %+v", f)
|
|
}
|
|
}
|
|
|
|
func TestProbeDraftErrorKinds(t *testing.T) {
|
|
status := func(code int) Server {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(code) }))
|
|
t.Cleanup(ts.Close)
|
|
return Server{URL: ts.URL + "/mcp"}
|
|
}
|
|
for name, tc := range map[string]struct {
|
|
server Server
|
|
want string
|
|
}{
|
|
"401": {status(http.StatusUnauthorized), "auth"},
|
|
"403": {status(http.StatusForbidden), "auth"},
|
|
"500": {status(http.StatusInternalServerError), "protocol"},
|
|
"unset variable": {Server{URL: "https://mcp.skillshare.invalid/mcp", BearerToken: &Value{FromEnv: "PROBE_DRAFT_UNSET"}}, "auth"},
|
|
"missing command": {Server{Command: "skillshare-probe-draft-missing"}, "command"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
got, err := ProbeDraft(tc.server, "", CheckOptions{Timeout: 5 * time.Second, LookupEnv: func(string) (string, bool) { return "", false }})
|
|
if err != nil || got.ErrorKind != tc.want || got.Error == "" {
|
|
t.Fatalf("got %+v, %v; want kind %s", got, err, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProbeDraftTimesOut(t *testing.T) {
|
|
release := make(chan struct{})
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-release }))
|
|
defer ts.Close()
|
|
defer close(release)
|
|
got, err := ProbeDraft(Server{URL: ts.URL + "/mcp"}, "", CheckOptions{Timeout: 200 * time.Millisecond})
|
|
if err != nil || got.ErrorKind != "timeout" {
|
|
t.Fatalf("got %+v, %v", got, err)
|
|
}
|
|
}
|