diff --git a/cmd/skillshare/upgrade.go b/cmd/skillshare/upgrade.go index 81417bcbb..bccfd0269 100644 --- a/cmd/skillshare/upgrade.go +++ b/cmd/skillshare/upgrade.go @@ -440,35 +440,18 @@ func downloadAndReplace(downloadURL, checksumsURL, destPath string, onProgress u } func downloadArchive(url string, onProgress utils.ProgressFunc) (string, error) { - client := &http.Client{Timeout: 5 * time.Minute} - resp, err := client.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return "", fmt.Errorf("download failed with status %d", resp.StatusCode) - } - tmp, err := os.CreateTemp("", "skillshare-upgrade-archive-*") if err != nil { return "", err } defer tmp.Close() - limited := io.LimitReader(utils.NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxArchiveSize+1) - n, err := io.Copy(tmp, limited) - if err != nil { + client := &http.Client{Timeout: 5 * time.Minute} + if err := utils.DownloadToFile(client, url, tmp, maxArchiveSize, onProgress); err != nil { tmp.Close() os.Remove(tmp.Name()) return "", err } - if n > maxArchiveSize { - tmp.Close() - os.Remove(tmp.Name()) - return "", fmt.Errorf("archive exceeds the maximum size (%d MB)", maxArchiveSize/(1024*1024)) - } return tmp.Name(), nil } diff --git a/internal/uidist/uidist.go b/internal/uidist/uidist.go index da63b4807..9eb4de40f 100644 --- a/internal/uidist/uidist.go +++ b/internal/uidist/uidist.go @@ -118,36 +118,18 @@ const maxDownloadSize = 100 * 1024 * 1024 // 100 MB // downloadToTemp downloads a URL to a temporary file and returns its path. func downloadToTemp(url string, onProgress utils.ProgressFunc) (string, error) { - client := &http.Client{Timeout: 120 * time.Second} - resp, err := client.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("download returned HTTP %d", resp.StatusCode) - } - tmp, err := os.CreateTemp("", "skillshare-ui-dist-*.tar.gz") if err != nil { return "", err } defer tmp.Close() - limited := io.LimitReader(utils.NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxDownloadSize+1) - n, err := io.Copy(tmp, limited) - if err != nil { + client := &http.Client{Timeout: 120 * time.Second} + if err := utils.DownloadToFile(client, url, tmp, maxDownloadSize, onProgress); err != nil { tmp.Close() os.Remove(tmp.Name()) return "", err } - if n > maxDownloadSize { - tmp.Close() - os.Remove(tmp.Name()) - return "", fmt.Errorf("download exceeds maximum size (%d MB)", maxDownloadSize/(1024*1024)) - } - return tmp.Name(), nil } diff --git a/internal/utils/download.go b/internal/utils/download.go new file mode 100644 index 000000000..51e377e8a --- /dev/null +++ b/internal/utils/download.go @@ -0,0 +1,183 @@ +package utils + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" +) + +// GitHub's release CDN throttles each connection (about 80 KB/s on some networks), +// so a large download is split into ranges fetched in parallel. +const ( + downloadParts = 8 + minParallelSize = 1 << 20 +) + +// DownloadToFile writes url into dst and rejects anything larger than maxSize. +// When the server supports range requests, a file of at least 1 MB downloads as +// parallel ranges; otherwise it streams as a single response. +func DownloadToFile(client *http.Client, url string, dst *os.File, maxSize int64, onProgress ProgressFunc) error { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + // A server without range support ignores this and sends the whole file with 200. + req.Header.Set("Range", "bytes=0-0") + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + limited := io.LimitReader(NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxSize+1) + n, err := io.Copy(dst, limited) + if err != nil { + return err + } + if n > maxSize { + return tooLarge(maxSize) + } + return nil + case http.StatusPartialContent: + total, err := rangeTotal(resp.Header.Get("Content-Range")) + if err != nil { + return err + } + if total > maxSize { + return tooLarge(maxSize) + } + parts := downloadParts + if total < minParallelSize { + parts = 1 + } + // Ranges go to the URL after redirects: the signed CDN URL, not github.com again. + return downloadRanges(client, resp.Request.URL.String(), dst, total, parts, onProgress) + default: + return fmt.Errorf("download failed with status %d", resp.StatusCode) + } +} + +func tooLarge(maxSize int64) error { + return fmt.Errorf("download exceeds the maximum size (%d MB)", maxSize/(1024*1024)) +} + +// rangeTotal reads the full size from a Content-Range header such as "bytes 0-0/9665580". +func rangeTotal(header string) (int64, error) { + _, size, ok := strings.Cut(header, "/") + total, err := strconv.ParseInt(size, 10, 64) + if !ok || err != nil || total <= 0 { + return 0, fmt.Errorf("download returned an invalid Content-Range %q", header) + } + return total, nil +} + +func downloadRanges(client *http.Client, url string, dst *os.File, total int64, parts int, onProgress ProgressFunc) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + progress := &sharedProgress{total: total, fn: onProgress} + + var wg sync.WaitGroup + var once sync.Once + var firstErr error + size := (total + int64(parts) - 1) / int64(parts) + for start := int64(0); start < total; start += size { + end := min(start+size, total) - 1 + wg.Add(1) + go func() { + defer wg.Done() + if err := downloadRange(ctx, rangeClient(client), url, dst, start, end, progress); err != nil { + once.Do(func() { firstErr = err; cancel() }) + } + }() + } + // Wait for every part, so no write lands after the caller removes a failed file. + wg.Wait() + return firstErr +} + +// rangeClient copies client with a clone of its transport, so each range gets its +// own connection: over HTTP/2 the ranges would share one, and the throttle is per +// connection. The clone keeps the caller's TLS, proxy and dialer settings. A +// transport that cannot be cloned is shared as is. +func rangeClient(client *http.Client) *http.Client { + c := *client + switch t := client.Transport.(type) { + case nil: + c.Transport = http.DefaultTransport.(*http.Transport).Clone() + case *http.Transport: + c.Transport = t.Clone() + } + return &c +} + +func downloadRange(ctx context.Context, client *http.Client, url string, dst *os.File, start, end int64, progress *sharedProgress) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + if t, ok := client.Transport.(*http.Transport); ok { + defer t.CloseIdleConnections() + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("download of bytes %d-%d failed with status %d", start, end, resp.StatusCode) + } + + want := end - start + 1 + body := io.LimitReader(resp.Body, want+1) + n, err := io.Copy(io.NewOffsetWriter(dst, start), &countingReader{r: body, add: progress.add}) + if err != nil { + return err + } + if n != want { + return fmt.Errorf("download of bytes %d-%d returned %d bytes", start, end, n) + } + return nil +} + +type countingReader struct { + r io.Reader + add func(int64) +} + +func (c *countingReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.add(int64(n)) + return n, err +} + +// sharedProgress sums the parts and reports like NewProgressReader: at most once +// per progressInterval, plus a final report when every byte has arrived. +type sharedProgress struct { + mu sync.Mutex + read int64 + total int64 + fn ProgressFunc + lastSent time.Time +} + +func (p *sharedProgress) add(n int64) { + if p.fn == nil || n == 0 { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.read += n + if p.read == p.total || time.Since(p.lastSent) >= progressInterval { + p.lastSent = time.Now() + p.fn(p.read, p.total) + } +} diff --git a/internal/utils/download_test.go b/internal/utils/download_test.go new file mode 100644 index 000000000..d5e3fe395 --- /dev/null +++ b/internal/utils/download_test.go @@ -0,0 +1,116 @@ +package utils + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" +) + +// payload is larger than minParallelSize and not a multiple of downloadParts. +var payload = bytes.Repeat([]byte("0123456789abcdef"), (minParallelSize+12345)/16) + +func download(t *testing.T, srv *httptest.Server, maxSize int64) ([]byte, error) { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "dl-*") + if err != nil { + t.Fatal(err) + } + defer f.Close() + client := &http.Client{Timeout: 10 * time.Second} + if err := DownloadToFile(client, srv.URL+"/asset", f, maxSize, nil); err != nil { + return nil, err + } + return os.ReadFile(f.Name()) +} + +func TestDownloadToFile_ParallelRangesReassembleExactBytes(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload)) + })) + defer srv.Close() + + got, err := download(t, srv, int64(len(payload))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload)) + } + if n := requests.Load(); n != 1+downloadParts { + t.Errorf("requests = %d, want a probe plus %d ranges", n, downloadParts) + } +} + +func TestDownloadToFile_ServerWithoutRangesStreamsWholeFile(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write(payload) + })) + defer srv.Close() + + got, err := download(t, srv, int64(len(payload))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload)) + } +} + +func TestDownloadToFile_FailedRangeReturnsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The probe and the first range succeed; every other range fails. + if !strings.HasPrefix(r.Header.Get("Range"), "bytes=0-") { + http.Error(w, "boom", http.StatusInternalServerError) + return + } + http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload)) + })) + defer srv.Close() + + if _, err := download(t, srv, int64(len(payload))); err == nil { + t.Fatal("expected an error when a range fails") + } +} + +func TestDownloadToFile_RejectsOversizeBeforeDownloading(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload)) + })) + defer srv.Close() + + _, err := download(t, srv, int64(len(payload))-1) + if err == nil || !strings.Contains(err.Error(), "exceeds the maximum size") { + t.Fatalf("err = %v, want the maximum size error", err) + } +} + +func TestDownloadToFile_RangesKeepTheClientTransport(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload)) + })) + defer srv.Close() + f, err := os.CreateTemp(t.TempDir(), "dl-*") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + // srv.Client() trusts the test server's certificate; http.DefaultTransport does not. + if err := DownloadToFile(srv.Client(), srv.URL+"/asset", f, int64(len(payload)), nil); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(f.Name()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload)) + } +} diff --git a/website/docs/reference/commands/ui.md b/website/docs/reference/commands/ui.md index 63cbf116d..21677e414 100644 --- a/website/docs/reference/commands/ui.md +++ b/website/docs/reference/commands/ui.md @@ -272,7 +272,7 @@ The dashboard reads and writes `.skillshare/config.yaml`, syncs to project-local `skillshare ui` automatically downloads pre-built UI assets from the matching GitHub Release on first launch. The assets are cached in `~/.cache/skillshare/ui//` (respects `XDG_CACHE_HOME`) so subsequent launches are instant and offline. -- **First run** requires an internet connection to download the UI assets (~1 MB) +- **First run** requires an internet connection to download the UI assets (about 2 MB) - **Subsequent runs** use the cached assets — no network needed - **On upgrade**, old cached versions are automatically cleaned up; the new UI is pre-downloaded during `skillshare upgrade` - **To clear the cache manually**, run `skillshare ui --clear-cache` diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/reference/commands/ui.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/reference/commands/ui.md index 783826478..0b5d84b6f 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/reference/commands/ui.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/reference/commands/ui.md @@ -270,7 +270,7 @@ skillshare ui -p `skillshare ui` は、初回起動時に対応する GitHub Release からビルド済みの UI アセットを自動的にダウンロードします。アセットは `~/.cache/skillshare/ui//`(`XDG_CACHE_HOME` を尊重)にキャッシュされるため、以降の起動は即座かつオフラインで行われます。 -- **初回実行** には UI アセット(約 1 MB)をダウンロードするためのインターネット接続が必要です +- **初回実行** には UI アセット(約 2 MB)をダウンロードするためのインターネット接続が必要です - **以降の実行** はキャッシュされたアセットを使用します — ネットワーク不要 - **アップグレード時**、古いキャッシュ済みバージョンは自動的にクリーンアップされます。新しい UI は `skillshare upgrade` の際に事前ダウンロードされます - **キャッシュを手動でクリアする** には `skillshare ui --clear-cache` を実行してください diff --git a/website/i18n/ko/docusaurus-plugin-content-docs/current/reference/commands/ui.md b/website/i18n/ko/docusaurus-plugin-content-docs/current/reference/commands/ui.md index 7037f04d8..fd77070ac 100644 --- a/website/i18n/ko/docusaurus-plugin-content-docs/current/reference/commands/ui.md +++ b/website/i18n/ko/docusaurus-plugin-content-docs/current/reference/commands/ui.md @@ -270,7 +270,7 @@ skillshare ui -p `skillshare ui`는 최초 실행 시 일치하는 GitHub Release에서 미리 빌드된 UI 자산을 자동으로 다운로드합니다. 자산은 `~/.cache/skillshare/ui//`에 캐시되어(`XDG_CACHE_HOME`을 따름) 이후 실행은 즉시, 오프라인으로 이루어집니다. -- **First run**은 UI 자산(~1MB)을 다운로드하기 위해 인터넷 연결이 필요합니다 +- **First run**은 UI 자산(약 2MB)을 다운로드하기 위해 인터넷 연결이 필요합니다 - **Subsequent runs**은 캐시된 자산을 사용합니다 — 네트워크가 필요 없습니다 - **On upgrade** 시, 이전에 캐시된 버전은 자동으로 정리되며, 새 UI는 `skillshare upgrade` 중에 미리 다운로드됩니다 - **To clear the cache manually**, `skillshare ui --clear-cache`를 실행하세요 diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/commands/ui.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/commands/ui.md index f56bf685a..65038dd73 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/commands/ui.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/commands/ui.md @@ -271,7 +271,7 @@ dashboard 会读写 `.skillshare/config.yaml`,同步到 project 本地的 targ `skillshare ui` 会在首次启动时自动从匹配的 GitHub Release 下载预构建的 UI 资源。这些资源会缓存在 `~/.cache/skillshare/ui//`(遵循 `XDG_CACHE_HOME`),因此后续启动是即时且离线的。 -- **首次运行** 需要网络连接来下载 UI 资源(约 1 MB) +- **首次运行** 需要网络连接来下载 UI 资源(约 2 MB) - **后续运行** 使用缓存的资源——无需网络 - **升级时**,旧的缓存版本会自动清理;新 UI 会在 `skillshare upgrade` 期间被预先下载 - **手动清除缓存**,运行 `skillshare ui --clear-cache` diff --git a/website/i18n/zh-Hant/docusaurus-plugin-content-docs/current/reference/commands/ui.md b/website/i18n/zh-Hant/docusaurus-plugin-content-docs/current/reference/commands/ui.md index 1c08997d4..c5987aff0 100644 --- a/website/i18n/zh-Hant/docusaurus-plugin-content-docs/current/reference/commands/ui.md +++ b/website/i18n/zh-Hant/docusaurus-plugin-content-docs/current/reference/commands/ui.md @@ -270,7 +270,7 @@ Dashboard 會讀寫 `.skillshare/config.yaml`、同步至 project 本機的 targ `skillshare ui` 會在首次啟動時,自動從對應的 GitHub Release 下載預先建置的 UI 資源。這些資源會快取於 `~/.cache/skillshare/ui//`(遵循 `XDG_CACHE_HOME`),因此後續啟動會是即時且離線的。 -- **首次執行**需要網際網路連線來下載 UI 資源(約 1 MB) +- **首次執行**需要網際網路連線來下載 UI 資源(約 2 MB) - **後續執行**使用快取的資源——不需要網路 - **升級時**,舊的快取版本會自動清除;新 UI 會在 `skillshare upgrade` 期間預先下載 - **手動清除快取**,執行 `skillshare ui --clear-cache`