diff --git a/internal/importmap/importmap_test.go b/internal/importmap/importmap_test.go index c6c5c6493..d01416cea 100644 --- a/internal/importmap/importmap_test.go +++ b/internal/importmap/importmap_test.go @@ -108,7 +108,10 @@ func TestAddPackages(t *testing.T) { func TestResolve(t *testing.T) { im := Blank() - im.AddImportFromSpecifier("react-dom@19/client", false) + _, errors := im.AddImportFromSpecifier("react-dom@19.2.4/client", false) + if len(errors) > 0 { + t.Fatalf("Failed to add react-dom/client: %v", errors) + } referrer, _ := url.Parse("file:///main.js") modUrl, ok := im.Resolve("react", referrer) if !ok { diff --git a/server/git.go b/server/git.go index 7f158b79e..8adb89612 100644 --- a/server/git.go +++ b/server/git.go @@ -3,6 +3,7 @@ package server import ( "bufio" "bytes" + "compress/gzip" "context" "errors" "fmt" @@ -10,12 +11,18 @@ import ( "net/url" "os" "os/exec" + "path/filepath" + "strings" "time" "github.com/esm-dev/esm.sh/internal/fetch" "github.com/ije/gox/utils" ) +const ghInstallTimeout = 30 * time.Second + +var errRepoTooLarge = errors.New("repo is too large") + type GitRef struct { Ref string Sha string @@ -69,12 +76,45 @@ func ghInstall(wd, name, tag string) (err error) { } func ghInstallContext(ctx context.Context, wd, name, tag string) (err error) { + if err := ctx.Err(); err != nil { + return err + } + tooLargeFile := filepath.Join(config.WorkDir, "gh-too-large", url.PathEscape(strings.ToLower(name))) + if existsFile(tooLargeFile) { + return errRepoTooLarge + } + + installCtx, cancel := context.WithTimeout(ctx, ghInstallTimeout) + defer cancel() + defer func() { + if err == nil { + err = installCtx.Err() + } + if errors.Is(err, errRepoTooLarge) { + recordErr := ensureDir(filepath.Dir(tooLargeFile)) + if recordErr == nil { + recordErr = os.WriteFile(tooLargeFile, []byte(name+"\n"), 0644) + } + if recordErr != nil { + err = errors.Join(err, fmt.Errorf("record oversized repo: %w", recordErr)) + } + } else if ctx.Err() != nil { + err = ctx.Err() + } else if errors.Is(installCtx.Err(), context.DeadlineExceeded) { + err = errors.New("github: install timeout after 30 seconds") + } + if err != nil { + // A partial extraction must not be treated as an installed package. + os.RemoveAll(wd) + } + }() + u, err := url.Parse(fmt.Sprintf("https://codeload.github.com/%s/tar.gz/%s", name, tag)) if err != nil { return } - client := fetch.NewClient("esmd/"+VERSION, 30, false) - res, err := client.FetchWithContext(ctx, u, nil) + client := fetch.NewClient("esmd/"+VERSION, 0, false) + res, err := client.FetchWithContext(installCtx, u, nil) if err != nil { return } @@ -88,11 +128,23 @@ func ghInstallContext(ctx context.Context, wd, name, tag string) (err error) { return fmt.Errorf("fetch %s failed: %s", u, res.Status) } - err = extractPackageTarballContext(ctx, wd, name, io.LimitReader(res.Body, maxPackageTarballSize)) + if res.ContentLength > maxPackageTarballSize { + return errRepoTooLarge + } + download := &io.LimitedReader{R: res.Body, N: maxPackageTarballSize + 1} + unzip, err := gzip.NewReader(&contextReader{ctx: installCtx, reader: download}) if err != nil { - // clear wd if failed to extract tarball, otherwise the partial - // extraction would be treated as a completed installation - os.RemoveAll(wd) + return err + } + defer unzip.Close() + unpacked := &io.LimitedReader{R: unzip, N: maxPackageTarballSize + 1} + err = extractPackageTarContext(installCtx, wd, name, unpacked) + if err == nil { + // Read through the gzip trailer and count any remaining archive data. + _, err = io.Copy(io.Discard, &contextReader{ctx: installCtx, reader: unpacked}) + } + if download.N == 0 || unpacked.N == 0 { + err = errRepoTooLarge } return } diff --git a/server/git_test.go b/server/git_test.go index 1b5ab7072..b14585d0a 100644 --- a/server/git_test.go +++ b/server/git_test.go @@ -1,14 +1,36 @@ package server import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "net/http" "os" "path" "path/filepath" + "strings" "testing" + "testing/synctest" + "time" "github.com/ije/gox/crypto/rand" ) +type ghTestTransport func(*http.Request) (*http.Response, error) + +func (f ghTestTransport) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + func TestListRepoRefs(t *testing.T) { refs, err := listGhRepoRefs("https://github.com/esm-dev/esm.sh") if err != nil { @@ -37,3 +59,158 @@ func TestGhInstall(t *testing.T) { t.Fatal("README.md not found") } } + +func TestGhInstallLimits(t *testing.T) { + workDir, transport := config.WorkDir, http.DefaultTransport + config.WorkDir = t.TempDir() + t.Cleanup(func() { + config.WorkDir, http.DefaultTransport = workDir, transport + }) + for _, test := range []struct { + name string + tooLarge bool + }{ + {"content-length", true}, + {"download", true}, + {"unpacked", true}, + {"valid", false}, + {"invalid", false}, + } { + t.Run(test.name, func(t *testing.T) { + requests := 0 + http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) { + requests++ + res := &http.Response{StatusCode: 200, Header: http.Header{}, ContentLength: -1} + if test.name == "content-length" { + res.ContentLength = maxPackageTarballSize + 1 + res.Body = io.NopCloser(strings.NewReader("")) + } else if test.name == "invalid" { + res.Body = io.NopCloser(strings.NewReader("invalid gzip")) + } else { + pr, pw := io.Pipe() + res.Body = pr + go func() { + defer pw.Close() + level := gzip.BestSpeed + if test.name == "download" { + level = gzip.NoCompression + } + gz, _ := gzip.NewWriterLevel(pw, level) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + if err := tw.WriteHeader(&tar.Header{Name: "repo/package.json", Mode: 0644, Size: 2}); err != nil { + return + } + if _, err := tw.Write([]byte("{}")); err != nil { + return + } + if test.tooLarge { + if err := tw.WriteHeader(&tar.Header{Name: "repo/large.bin", Mode: 0644, Size: maxPackageTarballSize + 1}); err != nil { + return + } + io.CopyN(tw, zeroReader{}, maxPackageTarballSize+1) + } + }() + } + return res, nil + }) + wd := filepath.Join(t.TempDir(), "install") + repo := "owner/" + test.name + err := ghInstall(wd, repo, "main") + if test.tooLarge { + if err != errRepoTooLarge || err.Error() != "repo is too large" { + t.Fatalf("expected repo size error, got %v", err) + } + if err := ghInstall(wd, strings.ToUpper(repo), "another-tag"); err != errRepoTooLarge { + t.Fatalf("expected recorded repo size error, got %v", err) + } + if requests != 1 { + t.Fatalf("recorded repo was fetched again: %d requests", requests) + } + } else if test.name == "valid" { + if err != nil { + t.Fatal(err) + } + if !existsFile(filepath.Join(wd, "node_modules", repo, "package.json")) { + t.Fatal("package.json not found") + } + } else if err == nil || errors.Is(err, errRepoTooLarge) { + t.Fatalf("expected invalid archive error, got %v", err) + } + if err != nil { + if _, statErr := os.Stat(wd); !os.IsNotExist(statErr) { + t.Fatalf("partial installation remains: %v", statErr) + } + } + }) + } + files, err := os.ReadDir(filepath.Join(config.WorkDir, "gh-too-large")) + if err != nil || len(files) != 3 { + t.Fatalf("expected three persistent repo records, got %d: %v", len(files), err) + } +} + +func TestGhInstallTimeout(t *testing.T) { + workDir, transport := config.WorkDir, http.DefaultTransport + config.WorkDir = t.TempDir() + t.Cleanup(func() { + config.WorkDir, http.DefaultTransport = workDir, transport + }) + for _, phase := range []string{"request", "body", "canceled", "parent-deadline"} { + t.Run(phase, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) { + if phase == "canceled" { + t.Fatal("canceled install made a request") + } + if phase != "body" { + <-r.Context().Done() + return nil, r.Context().Err() + } + pr, pw := io.Pipe() + go func() { + gz := gzip.NewWriter(pw) + tw := tar.NewWriter(gz) + tw.WriteHeader(&tar.Header{Name: "repo/package.json", Mode: 0644, Size: 2}) + tw.Write([]byte("{}")) + tw.Flush() + gz.Flush() + <-r.Context().Done() + pw.CloseWithError(r.Context().Err()) + }() + return &http.Response{StatusCode: 200, Header: http.Header{}, ContentLength: -1, Body: pr}, nil + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if phase == "canceled" { + cancel() + } else if phase == "parent-deadline" { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, time.Second) + defer stop() + } + wd := filepath.Join(t.TempDir(), "install") + start := time.Now() + err := ghInstallContext(ctx, wd, "owner/timeout", "main") + if phase == "canceled" { + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + } else if phase == "parent-deadline" { + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) != time.Second { + t.Fatalf("expected caller deadline, got %v after %v", err, time.Since(start)) + } + } else if err == nil || err.Error() != "github: install timeout after 30 seconds" || time.Since(start) != ghInstallTimeout { + t.Fatalf("expected install timeout, got %v after %v", err, time.Since(start)) + } + if _, err := os.Stat(wd); !os.IsNotExist(err) { + t.Fatalf("partial installation remains: %v", err) + } + }) + }) + } + if existsDir(filepath.Join(config.WorkDir, "gh-too-large")) { + t.Fatal("timed out repo was recorded as too large") + } +} diff --git a/server/npmrc.go b/server/npmrc.go index 17a0e3dfb..bec301351 100644 --- a/server/npmrc.go +++ b/server/npmrc.go @@ -718,15 +718,19 @@ func extractPackageTarball(installDir string, pkgName string, tarball io.Reader) } func extractPackageTarballContext(ctx context.Context, installDir string, pkgName string, tarball io.Reader) (err error) { - // pkgName is joined into the extraction path below. - if !filepath.IsLocal(pkgName) { - return errors.New("invalid package name: " + pkgName) - } - unziped, err := gzip.NewReader(&contextReader{ctx: ctx, reader: tarball}) if err != nil { return } + defer unziped.Close() + return extractPackageTarContext(ctx, installDir, pkgName, unziped) +} + +func extractPackageTarContext(ctx context.Context, installDir string, pkgName string, archive io.Reader) (err error) { + // pkgName is joined into the extraction path below. + if !filepath.IsLocal(pkgName) { + return errors.New("invalid package name: " + pkgName) + } // Confine every write to installDir, including when a symlink is already at // the destination. Lexical path checks alone cannot provide that guarantee. @@ -740,7 +744,7 @@ func extractPackageTarballContext(ctx context.Context, installDir string, pkgNam defer root.Close() // extract tarball - tr := tar.NewReader(unziped) + tr := tar.NewReader(&contextReader{ctx: ctx, reader: archive}) for { if err := ctx.Err(); err != nil { return err diff --git a/server/router.go b/server/router.go index dff3ed359..cd442c5ee 100644 --- a/server/router.go +++ b/server/router.go @@ -687,7 +687,7 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { } } - if pathKind == RawFile && !rawFlag && esmPath.SubPath != "" && strings.HasSuffix(esmPath.SubPath, ".map") { + if pathKind == RawFile && !esmPath.GhPrefix && !rawFlag && esmPath.SubPath != "" && strings.HasSuffix(esmPath.SubPath, ".map") { pkgJson, err := npmrc.installPackage(esmPath.Package()) if err != nil { writeStatus(w, 500, err.Error()) @@ -771,6 +771,78 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { } } + if esmPath.GhPrefix && hasTargetSegment && pathKind == RawFile && !rawFlag && esmPath.SubPath != "" { + assetURL := &url.URL{Path: "/" + esmPath.String(), RawQuery: r.URL.RawQuery} + redirect(w, origin+assetURL.String(), true) + return + } + + // Fetch GitHub assets directly without installing the repository. + if esmPath.GhPrefix && pathKind == RawFile && esmPath.SubPath != "" && !(query.Has("module") && endsWith(esmPath.SubPath, ".css", ".json", ".wasm")) { + rawURL := &url.URL{ + Scheme: "https", + Host: "raw.githubusercontent.com", + Path: "/" + esmPath.PkgName + "/" + esmPath.PkgVersion + "/" + esmPath.SubPath, + } + requestHeader := http.Header{} + if etag := r.Header.Get("If-None-Match"); etag != "" { + requestHeader.Set("If-None-Match", etag) + } + client := fetch.NewClient("esmd/"+VERSION, 30, false) + res, err := client.FetchWithContext(r.Context(), rawURL, requestHeader) + if err != nil { + writeStatus(w, 502, err.Error()) + return + } + defer res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 304 { + if res.StatusCode == 404 { + header.Set("Cache-Control", ccImmutable) + } + writeStatus(w, res.StatusCode, http.StatusText(res.StatusCode)) + return + } + if res.ContentLength > maxAssetFileSize { + header.Set("Cache-Control", ccImmutable) + writeStatus(w, 403, "File Too Large") + return + } + data, err := io.ReadAll(io.LimitReader(res.Body, maxAssetFileSize+1)) + if err != nil { + writeStatus(w, 502, err.Error()) + return + } + if len(data) > maxAssetFileSize { + header.Set("Cache-Control", ccImmutable) + writeStatus(w, 403, "File Too Large") + return + } + for _, key := range []string{"ETag", "Last-Modified"} { + if value := res.Header.Get(key); value != "" { + header.Set(key, value) + } + } + header.Set("Cache-Control", ccImmutable) + if res.StatusCode == 304 { + w.WriteHeader(304) + return + } + contentType := mime.GetContentType(esmPath.SubPath) + if endsWith(esmPath.SubPath, ".ts", ".mts", ".cts", ".tsx") { + contentType = ctTypeScript + } else if contentType == "" { + contentType = res.Header.Get("Content-Type") + } + header.Set("Content-Type", contentType) + if r.Method == http.MethodHead { + header.Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.WriteHeader(200) + return + } + writeBody(w, data) + return + } + // fix url that is related to `import.meta.url` if hasTargetSegment && isExactVersion && pathKind == RawFile && !rawFlag { extname := path.Ext(esmPath.SubPath) @@ -837,6 +909,12 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { // return css file as a `CSSStyleSheet` object when `?module` query is present if pathKind == RawFile && strings.HasSuffix(esmPath.SubPath, ".css") && query.Has("module") { + if esmPath.GhPrefix { + if _, err := npmrc.installPackageContext(r.Context(), esmPath.Package()); err != nil { + writeStatus(w, 500, err.Error()) + return + } + } filename := path.Join(npmrc.StoreDir(), esmPath.PackageId(), "node_modules", esmPath.PkgName, esmPath.SubPath) css, err := os.ReadFile(filename) if err != nil { diff --git a/server/router_test.go b/server/router_test.go index 95ad54481..bb8275de5 100644 --- a/server/router_test.go +++ b/server/router_test.go @@ -1,7 +1,17 @@ package server import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" + "time" + + "github.com/esm-dev/esm.sh/internal/storage" + "github.com/ije/gox/log" ) func TestCSSEntryRedirectURL(t *testing.T) { @@ -33,3 +43,146 @@ func TestCSSEntryRedirectURL(t *testing.T) { } } } + +func TestGhRawAssets(t *testing.T) { + previousConfig, previousNpmRC, transport := config, defaultNpmRC, http.DefaultTransport + testConfig := *config + testConfig.WorkDir = t.TempDir() + config, defaultNpmRC = &testConfig, nil + t.Cleanup(func() { + config, defaultNpmRC, http.DefaultTransport = previousConfig, previousNpmRC, transport + }) + markerDir := filepath.Join(config.WorkDir, "gh-too-large") + if err := os.MkdirAll(markerDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(markerDir, "owner%2Frepo"), []byte("owner/repo\n"), 0644); err != nil { + t.Fatal(err) + } + fs, err := storage.NewFSStorage(filepath.Join(config.WorkDir, "storage")) + if err != nil { + t.Fatal(err) + } + logger := new(log.Logger) + logger.SetOutput(io.Discard) + handler := esmRouter(fs, logger) + for _, test := range []struct { + name string + path string + method string + status int + contentType string + body string + }{ + {"svg", "assets/Alien/alien.svg", "GET", 200, "image/svg+xml; charset=utf-8", ""}, + {"escaped path", "assets/Alien%20%231.svg", "GET", 200, "image/svg+xml; charset=utf-8", ""}, + {"head", "alien.svg", "HEAD", 200, "image/svg+xml; charset=utf-8", ""}, + {"conditional", "alien.svg", "GET", 304, "", ""}, + {"css", "style.css", "GET", 200, ctCSS, "body { color: red }"}, + {"json", "data.json", "GET", 200, ctJSON, "{}"}, + {"source map", "index.js.map", "GET", 200, ctJSON, "{}"}, + {"raw relative asset", "es2022/alien.svg?raw", "GET", 200, "image/svg+xml; charset=utf-8", ""}, + {"raw typescript", "index.ts?raw", "GET", 200, ctTypeScript, "export default 1"}, + {"missing", "missing.svg", "GET", 404, "", "Not Found"}, + {"upstream error", "error.svg", "GET", 503, "", "Service Unavailable"}, + {"large content length", "large.svg", "GET", 403, "", "File Too Large"}, + {"large stream", "stream.svg", "GET", 403, "", "File Too Large"}, + {"javascript build", "index.js", "GET", 500, "", "repo is too large"}, + {"types build", "index.d.ts", "GET", 500, "", "Failed to build types: repo is too large"}, + {"json module", "data.json?module", "GET", 500, "", "repo is too large"}, + {"css module", "style.css?module", "GET", 500, "", "repo is too large"}, + } { + t.Run(test.name, func(t *testing.T) { + requests := 0 + http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) { + requests++ + if test.status == 500 || r.URL.Host != "raw.githubusercontent.com" || !strings.HasPrefix(r.URL.Path, "/owner/repo/abcdef0/") { + t.Errorf("unexpected upstream request: %s", r.URL) + } + if test.name == "escaped path" && (r.URL.EscapedPath() != "/owner/repo/abcdef0/assets/Alien%20%231.svg" || r.URL.RawQuery != "") { + t.Errorf("incorrect escaped path: %s", r.URL) + } + if test.name == "raw relative asset" && r.URL.Path != "/owner/repo/abcdef0/alien.svg" { + t.Errorf("incorrect relative asset path: %s", r.URL) + } + if deadline, ok := r.Context().Deadline(); !ok || time.Until(deadline) > 30*time.Second { + t.Error("expected a download deadline") + } + res := &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": {"text/plain"}, "Etag": {`"asset"`}}, + ContentLength: int64(len(test.body)), + Body: io.NopCloser(strings.NewReader(test.body)), + } + switch test.name { + case "conditional": + if r.Header.Get("If-None-Match") != `"asset"` { + t.Error("conditional header was not forwarded") + } + res.StatusCode = 304 + case "missing", "upstream error": + res.StatusCode = test.status + case "large content length": + res.ContentLength = maxAssetFileSize + 1 + case "large stream": + res.ContentLength = -1 + res.Body = io.NopCloser(io.LimitReader(zeroReader{}, maxAssetFileSize+1)) + } + return res, nil + }) + req := httptest.NewRequest(test.method, "http://localhost/gh/owner/repo@abcdef0/"+test.path, nil) + if test.name == "conditional" { + req.Header.Set("If-None-Match", `"asset"`) + } + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + if res.Code != test.status { + t.Fatalf("expected status %d, got %d: %s", test.status, res.Code, res.Body.String()) + } + if test.contentType != "" && res.Header().Get("Content-Type") != test.contentType { + t.Fatalf("unexpected content type: %s", res.Header().Get("Content-Type")) + } + if test.status == 200 || test.status == 304 { + if res.Header().Get("Cache-Control") != ccImmutable || res.Header().Get("Etag") != `"asset"` { + t.Fatalf("missing cache headers: %v", res.Header()) + } + } + body := test.body + if test.method == "HEAD" { + body = "" + } + if res.Body.String() != body { + t.Fatalf("unexpected response body: %q", res.Body.String()) + } + if test.status == 500 && requests != 0 || test.status != 500 && requests != 1 { + t.Fatalf("unexpected upstream request count: %d", requests) + } + }) + } + http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) { + t.Errorf("redirect made an upstream request: %s", r.URL) + return &http.Response{StatusCode: 404, Body: http.NoBody}, nil + }) + for _, test := range []struct{ path, location string }{ + {"es2022/alien.svg", "alien.svg"}, + {"es2022/napi/parser/parser.wasm32-wasi.wasm", "napi/parser/parser.wasm32-wasi.wasm"}, + {"X-ZHJlYWN0QDE4LjMuMQ/es2022/napi/parser/parser.wasm32-wasi.wasm", "napi/parser/parser.wasm32-wasi.wasm"}, + {"es2022/assets/Alien%20%231.svg?key=a%2Bb", "assets/Alien%20%231.svg?key=a%2Bb"}, + {"es2022/data.json?module", "data.json?module"}, + } { + t.Run("redirect/"+test.path, func(t *testing.T) { + base := "http://localhost/gh/owner/repo@abcdef0/" + res := httptest.NewRecorder() + handler.ServeHTTP(res, httptest.NewRequest("GET", base+test.path, nil)) + if res.Code != 301 || res.Header().Get("Location") != base+test.location { + t.Fatalf("unexpected redirect: %d %s", res.Code, res.Header().Get("Location")) + } + if res.Header().Get("Cache-Control") != ccImmutable { + t.Fatal("redirect is not immutable") + } + }) + } + if existsDir(filepath.Join(config.WorkDir, "npm", "gh", "owner")) { + t.Fatal("asset request installed the repository") + } +}