mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-18 18:04:17 +00:00
perf(upgrade): download release assets over parallel ranges (#381)
* perf(upgrade): download release assets over parallel ranges release-assets.githubusercontent.com throttles each connection on some networks. The 9.6 MB darwin_arm64 archive took 122-171 s over one connection but 25 s as 8 ranges, each on its own connection, with an identical checksum. `skillshare upgrade` and the first `skillshare ui` of a new version both waited minutes on that download. DownloadToFile probes with `Range: bytes=0-0`; a 206 answer splits the file into ranges fetched in parallel from the redirected URL, and a 200 answer streams the single response as before, so servers without range support cost no extra request. Size limits and checksum verification are unchanged. The UI asset download is about 2 MB now, so the docs no longer say 1 MB. * fix(upgrade): keep the caller's transport for parallel ranges Each range built its client from http.DefaultTransport, so a caller's custom transport (private CA, authenticated proxy, custom dialer) only applied to the probe. The ranges then failed or bypassed the caller's network setup even though the probe succeeded. Each range now copies the caller's client with a clone of its *http.Transport, which keeps those settings and still gives the range its own connection. A transport that cannot be cloned is shared as is.
This commit is contained in:
@@ -440,35 +440,18 @@ func downloadAndReplace(downloadURL, checksumsURL, destPath string, onProgress u
|
||||
}
|
||||
|
||||
func downloadArchive(url string, onProgress utils.ProgressFunc) (string, error) {
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("download failed with status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "skillshare-upgrade-archive-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tmp.Close()
|
||||
|
||||
limited := io.LimitReader(utils.NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxArchiveSize+1)
|
||||
n, err := io.Copy(tmp, limited)
|
||||
if err != nil {
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
if err := utils.DownloadToFile(client, url, tmp, maxArchiveSize, onProgress); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
return "", err
|
||||
}
|
||||
if n > maxArchiveSize {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
return "", fmt.Errorf("archive exceeds the maximum size (%d MB)", maxArchiveSize/(1024*1024))
|
||||
}
|
||||
return tmp.Name(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -118,36 +118,18 @@ const maxDownloadSize = 100 * 1024 * 1024 // 100 MB
|
||||
|
||||
// downloadToTemp downloads a URL to a temporary file and returns its path.
|
||||
func downloadToTemp(url string, onProgress utils.ProgressFunc) (string, error) {
|
||||
client := &http.Client{Timeout: 120 * time.Second}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("download returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "skillshare-ui-dist-*.tar.gz")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tmp.Close()
|
||||
|
||||
limited := io.LimitReader(utils.NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxDownloadSize+1)
|
||||
n, err := io.Copy(tmp, limited)
|
||||
if err != nil {
|
||||
client := &http.Client{Timeout: 120 * time.Second}
|
||||
if err := utils.DownloadToFile(client, url, tmp, maxDownloadSize, onProgress); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
return "", err
|
||||
}
|
||||
if n > maxDownloadSize {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
return "", fmt.Errorf("download exceeds maximum size (%d MB)", maxDownloadSize/(1024*1024))
|
||||
}
|
||||
|
||||
return tmp.Name(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GitHub's release CDN throttles each connection (about 80 KB/s on some networks),
|
||||
// so a large download is split into ranges fetched in parallel.
|
||||
const (
|
||||
downloadParts = 8
|
||||
minParallelSize = 1 << 20
|
||||
)
|
||||
|
||||
// DownloadToFile writes url into dst and rejects anything larger than maxSize.
|
||||
// When the server supports range requests, a file of at least 1 MB downloads as
|
||||
// parallel ranges; otherwise it streams as a single response.
|
||||
func DownloadToFile(client *http.Client, url string, dst *os.File, maxSize int64, onProgress ProgressFunc) error {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A server without range support ignores this and sends the whole file with 200.
|
||||
req.Header.Set("Range", "bytes=0-0")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
limited := io.LimitReader(NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxSize+1)
|
||||
n, err := io.Copy(dst, limited)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > maxSize {
|
||||
return tooLarge(maxSize)
|
||||
}
|
||||
return nil
|
||||
case http.StatusPartialContent:
|
||||
total, err := rangeTotal(resp.Header.Get("Content-Range"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if total > maxSize {
|
||||
return tooLarge(maxSize)
|
||||
}
|
||||
parts := downloadParts
|
||||
if total < minParallelSize {
|
||||
parts = 1
|
||||
}
|
||||
// Ranges go to the URL after redirects: the signed CDN URL, not github.com again.
|
||||
return downloadRanges(client, resp.Request.URL.String(), dst, total, parts, onProgress)
|
||||
default:
|
||||
return fmt.Errorf("download failed with status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func tooLarge(maxSize int64) error {
|
||||
return fmt.Errorf("download exceeds the maximum size (%d MB)", maxSize/(1024*1024))
|
||||
}
|
||||
|
||||
// rangeTotal reads the full size from a Content-Range header such as "bytes 0-0/9665580".
|
||||
func rangeTotal(header string) (int64, error) {
|
||||
_, size, ok := strings.Cut(header, "/")
|
||||
total, err := strconv.ParseInt(size, 10, 64)
|
||||
if !ok || err != nil || total <= 0 {
|
||||
return 0, fmt.Errorf("download returned an invalid Content-Range %q", header)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func downloadRanges(client *http.Client, url string, dst *os.File, total int64, parts int, onProgress ProgressFunc) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
progress := &sharedProgress{total: total, fn: onProgress}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var once sync.Once
|
||||
var firstErr error
|
||||
size := (total + int64(parts) - 1) / int64(parts)
|
||||
for start := int64(0); start < total; start += size {
|
||||
end := min(start+size, total) - 1
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := downloadRange(ctx, rangeClient(client), url, dst, start, end, progress); err != nil {
|
||||
once.Do(func() { firstErr = err; cancel() })
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Wait for every part, so no write lands after the caller removes a failed file.
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// rangeClient copies client with a clone of its transport, so each range gets its
|
||||
// own connection: over HTTP/2 the ranges would share one, and the throttle is per
|
||||
// connection. The clone keeps the caller's TLS, proxy and dialer settings. A
|
||||
// transport that cannot be cloned is shared as is.
|
||||
func rangeClient(client *http.Client) *http.Client {
|
||||
c := *client
|
||||
switch t := client.Transport.(type) {
|
||||
case nil:
|
||||
c.Transport = http.DefaultTransport.(*http.Transport).Clone()
|
||||
case *http.Transport:
|
||||
c.Transport = t.Clone()
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
func downloadRange(ctx context.Context, client *http.Client, url string, dst *os.File, start, end int64, progress *sharedProgress) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
if t, ok := client.Transport.(*http.Transport); ok {
|
||||
defer t.CloseIdleConnections()
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return fmt.Errorf("download of bytes %d-%d failed with status %d", start, end, resp.StatusCode)
|
||||
}
|
||||
|
||||
want := end - start + 1
|
||||
body := io.LimitReader(resp.Body, want+1)
|
||||
n, err := io.Copy(io.NewOffsetWriter(dst, start), &countingReader{r: body, add: progress.add})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != want {
|
||||
return fmt.Errorf("download of bytes %d-%d returned %d bytes", start, end, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
add func(int64)
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(b []byte) (int, error) {
|
||||
n, err := c.r.Read(b)
|
||||
c.add(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
// sharedProgress sums the parts and reports like NewProgressReader: at most once
|
||||
// per progressInterval, plus a final report when every byte has arrived.
|
||||
type sharedProgress struct {
|
||||
mu sync.Mutex
|
||||
read int64
|
||||
total int64
|
||||
fn ProgressFunc
|
||||
lastSent time.Time
|
||||
}
|
||||
|
||||
func (p *sharedProgress) add(n int64) {
|
||||
if p.fn == nil || n == 0 {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.read += n
|
||||
if p.read == p.total || time.Since(p.lastSent) >= progressInterval {
|
||||
p.lastSent = time.Now()
|
||||
p.fn(p.read, p.total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// payload is larger than minParallelSize and not a multiple of downloadParts.
|
||||
var payload = bytes.Repeat([]byte("0123456789abcdef"), (minParallelSize+12345)/16)
|
||||
|
||||
func download(t *testing.T, srv *httptest.Server, maxSize int64) ([]byte, error) {
|
||||
t.Helper()
|
||||
f, err := os.CreateTemp(t.TempDir(), "dl-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
if err := DownloadToFile(client, srv.URL+"/asset", f, maxSize, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return os.ReadFile(f.Name())
|
||||
}
|
||||
|
||||
func TestDownloadToFile_ParallelRangesReassembleExactBytes(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
got, err := download(t, srv, int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload))
|
||||
}
|
||||
if n := requests.Load(); n != 1+downloadParts {
|
||||
t.Errorf("requests = %d, want a probe plus %d ranges", n, downloadParts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadToFile_ServerWithoutRangesStreamsWholeFile(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
got, err := download(t, srv, int64(len(payload)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadToFile_FailedRangeReturnsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// The probe and the first range succeed; every other range fails.
|
||||
if !strings.HasPrefix(r.Header.Get("Range"), "bytes=0-") {
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := download(t, srv, int64(len(payload))); err == nil {
|
||||
t.Fatal("expected an error when a range fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadToFile_RejectsOversizeBeforeDownloading(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := download(t, srv, int64(len(payload))-1)
|
||||
if err == nil || !strings.Contains(err.Error(), "exceeds the maximum size") {
|
||||
t.Fatalf("err = %v, want the maximum size error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadToFile_RangesKeepTheClientTransport(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(w, r, "asset", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer srv.Close()
|
||||
f, err := os.CreateTemp(t.TempDir(), "dl-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// srv.Client() trusts the test server's certificate; http.DefaultTransport does not.
|
||||
if err := DownloadToFile(srv.Client(), srv.URL+"/asset", f, int64(len(payload)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.ReadFile(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("got %d bytes, want the exact %d-byte payload", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
@@ -272,7 +272,7 @@ The dashboard reads and writes `.skillshare/config.yaml`, syncs to project-local
|
||||
|
||||
`skillshare ui` automatically downloads pre-built UI assets from the matching GitHub Release on first launch. The assets are cached in `~/.cache/skillshare/ui/<version>/` (respects `XDG_CACHE_HOME`) so subsequent launches are instant and offline.
|
||||
|
||||
- **First run** requires an internet connection to download the UI assets (~1 MB)
|
||||
- **First run** requires an internet connection to download the UI assets (about 2 MB)
|
||||
- **Subsequent runs** use the cached assets — no network needed
|
||||
- **On upgrade**, old cached versions are automatically cleaned up; the new UI is pre-downloaded during `skillshare upgrade`
|
||||
- **To clear the cache manually**, run `skillshare ui --clear-cache`
|
||||
|
||||
@@ -270,7 +270,7 @@ skillshare ui -p
|
||||
|
||||
`skillshare ui` は、初回起動時に対応する GitHub Release からビルド済みの UI アセットを自動的にダウンロードします。アセットは `~/.cache/skillshare/ui/<version>/`(`XDG_CACHE_HOME` を尊重)にキャッシュされるため、以降の起動は即座かつオフラインで行われます。
|
||||
|
||||
- **初回実行** には UI アセット(約 1 MB)をダウンロードするためのインターネット接続が必要です
|
||||
- **初回実行** には UI アセット(約 2 MB)をダウンロードするためのインターネット接続が必要です
|
||||
- **以降の実行** はキャッシュされたアセットを使用します — ネットワーク不要
|
||||
- **アップグレード時**、古いキャッシュ済みバージョンは自動的にクリーンアップされます。新しい UI は `skillshare upgrade` の際に事前ダウンロードされます
|
||||
- **キャッシュを手動でクリアする** には `skillshare ui --clear-cache` を実行してください
|
||||
|
||||
@@ -270,7 +270,7 @@ skillshare ui -p
|
||||
|
||||
`skillshare ui`는 최초 실행 시 일치하는 GitHub Release에서 미리 빌드된 UI 자산을 자동으로 다운로드합니다. 자산은 `~/.cache/skillshare/ui/<version>/`에 캐시되어(`XDG_CACHE_HOME`을 따름) 이후 실행은 즉시, 오프라인으로 이루어집니다.
|
||||
|
||||
- **First run**은 UI 자산(~1MB)을 다운로드하기 위해 인터넷 연결이 필요합니다
|
||||
- **First run**은 UI 자산(약 2MB)을 다운로드하기 위해 인터넷 연결이 필요합니다
|
||||
- **Subsequent runs**은 캐시된 자산을 사용합니다 — 네트워크가 필요 없습니다
|
||||
- **On upgrade** 시, 이전에 캐시된 버전은 자동으로 정리되며, 새 UI는 `skillshare upgrade` 중에 미리 다운로드됩니다
|
||||
- **To clear the cache manually**, `skillshare ui --clear-cache`를 실행하세요
|
||||
|
||||
+1
-1
@@ -271,7 +271,7 @@ dashboard 会读写 `.skillshare/config.yaml`,同步到 project 本地的 targ
|
||||
|
||||
`skillshare ui` 会在首次启动时自动从匹配的 GitHub Release 下载预构建的 UI 资源。这些资源会缓存在 `~/.cache/skillshare/ui/<version>/`(遵循 `XDG_CACHE_HOME`),因此后续启动是即时且离线的。
|
||||
|
||||
- **首次运行** 需要网络连接来下载 UI 资源(约 1 MB)
|
||||
- **首次运行** 需要网络连接来下载 UI 资源(约 2 MB)
|
||||
- **后续运行** 使用缓存的资源——无需网络
|
||||
- **升级时**,旧的缓存版本会自动清理;新 UI 会在 `skillshare upgrade` 期间被预先下载
|
||||
- **手动清除缓存**,运行 `skillshare ui --clear-cache`
|
||||
|
||||
+1
-1
@@ -270,7 +270,7 @@ Dashboard 會讀寫 `.skillshare/config.yaml`、同步至 project 本機的 targ
|
||||
|
||||
`skillshare ui` 會在首次啟動時,自動從對應的 GitHub Release 下載預先建置的 UI 資源。這些資源會快取於 `~/.cache/skillshare/ui/<version>/`(遵循 `XDG_CACHE_HOME`),因此後續啟動會是即時且離線的。
|
||||
|
||||
- **首次執行**需要網際網路連線來下載 UI 資源(約 1 MB)
|
||||
- **首次執行**需要網際網路連線來下載 UI 資源(約 2 MB)
|
||||
- **後續執行**使用快取的資源——不需要網路
|
||||
- **升級時**,舊的快取版本會自動清除;新 UI 會在 `skillshare upgrade` 期間預先下載
|
||||
- **手動清除快取**,執行 `skillshare ui --clear-cache`
|
||||
|
||||
Reference in New Issue
Block a user