mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-20 20:34:19 +00:00
* perf(upgrade): download release assets over parallel ranges release-assets.githubusercontent.com throttles each connection on some networks. The 9.6 MB darwin_arm64 archive took 122-171 s over one connection but 25 s as 8 ranges, each on its own connection, with an identical checksum. `skillshare upgrade` and the first `skillshare ui` of a new version both waited minutes on that download. DownloadToFile probes with `Range: bytes=0-0`; a 206 answer splits the file into ranges fetched in parallel from the redirected URL, and a 200 answer streams the single response as before, so servers without range support cost no extra request. Size limits and checksum verification are unchanged. The UI asset download is about 2 MB now, so the docs no longer say 1 MB. * fix(upgrade): keep the caller's transport for parallel ranges Each range built its client from http.DefaultTransport, so a caller's custom transport (private CA, authenticated proxy, custom dialer) only applied to the probe. The ranges then failed or bypassed the caller's network setup even though the probe succeeded. Each range now copies the caller's client with a clone of its *http.Transport, which keeps those settings and still gives the range its own connection. A transport that cannot be cloned is shared as is.
184 lines
5.1 KiB
Go
184 lines
5.1 KiB
Go
package utils
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// GitHub's release CDN throttles each connection (about 80 KB/s on some networks),
|
|
// so a large download is split into ranges fetched in parallel.
|
|
const (
|
|
downloadParts = 8
|
|
minParallelSize = 1 << 20
|
|
)
|
|
|
|
// DownloadToFile writes url into dst and rejects anything larger than maxSize.
|
|
// When the server supports range requests, a file of at least 1 MB downloads as
|
|
// parallel ranges; otherwise it streams as a single response.
|
|
func DownloadToFile(client *http.Client, url string, dst *os.File, maxSize int64, onProgress ProgressFunc) error {
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// A server without range support ignores this and sends the whole file with 200.
|
|
req.Header.Set("Range", "bytes=0-0")
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
switch resp.StatusCode {
|
|
case http.StatusOK:
|
|
limited := io.LimitReader(NewProgressReader(resp.Body, resp.ContentLength, onProgress), maxSize+1)
|
|
n, err := io.Copy(dst, limited)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > maxSize {
|
|
return tooLarge(maxSize)
|
|
}
|
|
return nil
|
|
case http.StatusPartialContent:
|
|
total, err := rangeTotal(resp.Header.Get("Content-Range"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if total > maxSize {
|
|
return tooLarge(maxSize)
|
|
}
|
|
parts := downloadParts
|
|
if total < minParallelSize {
|
|
parts = 1
|
|
}
|
|
// Ranges go to the URL after redirects: the signed CDN URL, not github.com again.
|
|
return downloadRanges(client, resp.Request.URL.String(), dst, total, parts, onProgress)
|
|
default:
|
|
return fmt.Errorf("download failed with status %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func tooLarge(maxSize int64) error {
|
|
return fmt.Errorf("download exceeds the maximum size (%d MB)", maxSize/(1024*1024))
|
|
}
|
|
|
|
// rangeTotal reads the full size from a Content-Range header such as "bytes 0-0/9665580".
|
|
func rangeTotal(header string) (int64, error) {
|
|
_, size, ok := strings.Cut(header, "/")
|
|
total, err := strconv.ParseInt(size, 10, 64)
|
|
if !ok || err != nil || total <= 0 {
|
|
return 0, fmt.Errorf("download returned an invalid Content-Range %q", header)
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func downloadRanges(client *http.Client, url string, dst *os.File, total int64, parts int, onProgress ProgressFunc) error {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
progress := &sharedProgress{total: total, fn: onProgress}
|
|
|
|
var wg sync.WaitGroup
|
|
var once sync.Once
|
|
var firstErr error
|
|
size := (total + int64(parts) - 1) / int64(parts)
|
|
for start := int64(0); start < total; start += size {
|
|
end := min(start+size, total) - 1
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := downloadRange(ctx, rangeClient(client), url, dst, start, end, progress); err != nil {
|
|
once.Do(func() { firstErr = err; cancel() })
|
|
}
|
|
}()
|
|
}
|
|
// Wait for every part, so no write lands after the caller removes a failed file.
|
|
wg.Wait()
|
|
return firstErr
|
|
}
|
|
|
|
// rangeClient copies client with a clone of its transport, so each range gets its
|
|
// own connection: over HTTP/2 the ranges would share one, and the throttle is per
|
|
// connection. The clone keeps the caller's TLS, proxy and dialer settings. A
|
|
// transport that cannot be cloned is shared as is.
|
|
func rangeClient(client *http.Client) *http.Client {
|
|
c := *client
|
|
switch t := client.Transport.(type) {
|
|
case nil:
|
|
c.Transport = http.DefaultTransport.(*http.Transport).Clone()
|
|
case *http.Transport:
|
|
c.Transport = t.Clone()
|
|
}
|
|
return &c
|
|
}
|
|
|
|
func downloadRange(ctx context.Context, client *http.Client, url string, dst *os.File, start, end int64, progress *sharedProgress) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
|
if t, ok := client.Transport.(*http.Transport); ok {
|
|
defer t.CloseIdleConnections()
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusPartialContent {
|
|
return fmt.Errorf("download of bytes %d-%d failed with status %d", start, end, resp.StatusCode)
|
|
}
|
|
|
|
want := end - start + 1
|
|
body := io.LimitReader(resp.Body, want+1)
|
|
n, err := io.Copy(io.NewOffsetWriter(dst, start), &countingReader{r: body, add: progress.add})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != want {
|
|
return fmt.Errorf("download of bytes %d-%d returned %d bytes", start, end, n)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type countingReader struct {
|
|
r io.Reader
|
|
add func(int64)
|
|
}
|
|
|
|
func (c *countingReader) Read(b []byte) (int, error) {
|
|
n, err := c.r.Read(b)
|
|
c.add(int64(n))
|
|
return n, err
|
|
}
|
|
|
|
// sharedProgress sums the parts and reports like NewProgressReader: at most once
|
|
// per progressInterval, plus a final report when every byte has arrived.
|
|
type sharedProgress struct {
|
|
mu sync.Mutex
|
|
read int64
|
|
total int64
|
|
fn ProgressFunc
|
|
lastSent time.Time
|
|
}
|
|
|
|
func (p *sharedProgress) add(n int64) {
|
|
if p.fn == nil || n == 0 {
|
|
return
|
|
}
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.read += n
|
|
if p.read == p.total || time.Since(p.lastSent) >= progressInterval {
|
|
p.lastSent = time.Now()
|
|
p.fn(p.read, p.total)
|
|
}
|
|
}
|