Limit GitHub installs and fetch assets directly

internal/importmap/importmap_test.go

@@ -108,7 +108,10 @@ func TestAddPackages(t *testing.T) {
 
 func TestResolve(t *testing.T) {
 	im := Blank()
-	im.AddImportFromSpecifier("react-dom@19/client", false)
+	_, errors := im.AddImportFromSpecifier("react-dom@19.2.4/client", false)
+	if len(errors) > 0 {
+		t.Fatalf("Failed to add react-dom/client: %v", errors)
+	}
 	referrer, _ := url.Parse("file:///main.js")
 	modUrl, ok := im.Resolve("react", referrer)
 	if !ok {

server/git.go

@@ -3,19 +3,26 @@ package server
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
 	"io"
 	"net/url"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/esm-dev/esm.sh/internal/fetch"
 	"github.com/ije/gox/utils"
 )
 
+const ghInstallTimeout = 30 * time.Second
+
+var errRepoTooLarge = errors.New("repo is too large")
+
 type GitRef struct {
 	Ref string
 	Sha string
@@ -69,12 +76,45 @@ func ghInstall(wd, name, tag string) (err error) {
 }
 
 func ghInstallContext(ctx context.Context, wd, name, tag string) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	tooLargeFile := filepath.Join(config.WorkDir, "gh-too-large", url.PathEscape(strings.ToLower(name)))
+	if existsFile(tooLargeFile) {
+		return errRepoTooLarge
+	}
+
+	installCtx, cancel := context.WithTimeout(ctx, ghInstallTimeout)
+	defer cancel()
+	defer func() {
+		if err == nil {
+			err = installCtx.Err()
+		}
+		if errors.Is(err, errRepoTooLarge) {
+			recordErr := ensureDir(filepath.Dir(tooLargeFile))
+			if recordErr == nil {
+				recordErr = os.WriteFile(tooLargeFile, []byte(name+"\n"), 0644)
+			}
+			if recordErr != nil {
+				err = errors.Join(err, fmt.Errorf("record oversized repo: %w", recordErr))
+			}
+		} else if ctx.Err() != nil {
+			err = ctx.Err()
+		} else if errors.Is(installCtx.Err(), context.DeadlineExceeded) {
+			err = errors.New("github: install timeout after 30 seconds")
+		}
+		if err != nil {
+			// A partial extraction must not be treated as an installed package.
+			os.RemoveAll(wd)
+		}
+	}()
+
 	u, err := url.Parse(fmt.Sprintf("https://codeload.github.com/%s/tar.gz/%s", name, tag))
 	if err != nil {
 		return
 	}
-	client := fetch.NewClient("esmd/"+VERSION, 30, false)
-	res, err := client.FetchWithContext(ctx, u, nil)
+	client := fetch.NewClient("esmd/"+VERSION, 0, false)
+	res, err := client.FetchWithContext(installCtx, u, nil)
 	if err != nil {
 		return
 	}
@@ -88,11 +128,23 @@ func ghInstallContext(ctx context.Context, wd, name, tag string) (err error) {
 		return fmt.Errorf("fetch %s failed: %s", u, res.Status)
 	}
 
-	err = extractPackageTarballContext(ctx, wd, name, io.LimitReader(res.Body, maxPackageTarballSize))
+	if res.ContentLength > maxPackageTarballSize {
+		return errRepoTooLarge
+	}
+	download := &io.LimitedReader{R: res.Body, N: maxPackageTarballSize + 1}
+	unzip, err := gzip.NewReader(&contextReader{ctx: installCtx, reader: download})
 	if err != nil {
-		// clear wd if failed to extract tarball, otherwise the partial
-		// extraction would be treated as a completed installation
-		os.RemoveAll(wd)
+		return err
+	}
+	defer unzip.Close()
+	unpacked := &io.LimitedReader{R: unzip, N: maxPackageTarballSize + 1}
+	err = extractPackageTarContext(installCtx, wd, name, unpacked)
+	if err == nil {
+		// Read through the gzip trailer and count any remaining archive data.
+		_, err = io.Copy(io.Discard, &contextReader{ctx: installCtx, reader: unpacked})
+	}
+	if download.N == 0 || unpacked.N == 0 {
+		err = errRepoTooLarge
 	}
 	return
 }

server/git_test.go

@@ -1,14 +1,36 @@
 package server
 
 import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"testing"
+	"testing/synctest"
+	"time"
 
 	"github.com/ije/gox/crypto/rand"
 )
 
+type ghTestTransport func(*http.Request) (*http.Response, error)
+
+func (f ghTestTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+type zeroReader struct{}
+
+func (zeroReader) Read(p []byte) (int, error) {
+	clear(p)
+	return len(p), nil
+}
+
 func TestListRepoRefs(t *testing.T) {
 	refs, err := listGhRepoRefs("https://github.com/esm-dev/esm.sh")
 	if err != nil {
@@ -37,3 +59,158 @@ func TestGhInstall(t *testing.T) {
 		t.Fatal("README.md not found")
 	}
 }
+
+func TestGhInstallLimits(t *testing.T) {
+	workDir, transport := config.WorkDir, http.DefaultTransport
+	config.WorkDir = t.TempDir()
+	t.Cleanup(func() {
+		config.WorkDir, http.DefaultTransport = workDir, transport
+	})
+	for _, test := range []struct {
+		name     string
+		tooLarge bool
+	}{
+		{"content-length", true},
+		{"download", true},
+		{"unpacked", true},
+		{"valid", false},
+		{"invalid", false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			requests := 0
+			http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) {
+				requests++
+				res := &http.Response{StatusCode: 200, Header: http.Header{}, ContentLength: -1}
+				if test.name == "content-length" {
+					res.ContentLength = maxPackageTarballSize + 1
+					res.Body = io.NopCloser(strings.NewReader(""))
+				} else if test.name == "invalid" {
+					res.Body = io.NopCloser(strings.NewReader("invalid gzip"))
+				} else {
+					pr, pw := io.Pipe()
+					res.Body = pr
+					go func() {
+						defer pw.Close()
+						level := gzip.BestSpeed
+						if test.name == "download" {
+							level = gzip.NoCompression
+						}
+						gz, _ := gzip.NewWriterLevel(pw, level)
+						defer gz.Close()
+						tw := tar.NewWriter(gz)
+						defer tw.Close()
+						if err := tw.WriteHeader(&tar.Header{Name: "repo/package.json", Mode: 0644, Size: 2}); err != nil {
+							return
+						}
+						if _, err := tw.Write([]byte("{}")); err != nil {
+							return
+						}
+						if test.tooLarge {
+							if err := tw.WriteHeader(&tar.Header{Name: "repo/large.bin", Mode: 0644, Size: maxPackageTarballSize + 1}); err != nil {
+								return
+							}
+							io.CopyN(tw, zeroReader{}, maxPackageTarballSize+1)
+						}
+					}()
+				}
+				return res, nil
+			})
+			wd := filepath.Join(t.TempDir(), "install")
+			repo := "owner/" + test.name
+			err := ghInstall(wd, repo, "main")
+			if test.tooLarge {
+				if err != errRepoTooLarge || err.Error() != "repo is too large" {
+					t.Fatalf("expected repo size error, got %v", err)
+				}
+				if err := ghInstall(wd, strings.ToUpper(repo), "another-tag"); err != errRepoTooLarge {
+					t.Fatalf("expected recorded repo size error, got %v", err)
+				}
+				if requests != 1 {
+					t.Fatalf("recorded repo was fetched again: %d requests", requests)
+				}
+			} else if test.name == "valid" {
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !existsFile(filepath.Join(wd, "node_modules", repo, "package.json")) {
+					t.Fatal("package.json not found")
+				}
+			} else if err == nil || errors.Is(err, errRepoTooLarge) {
+				t.Fatalf("expected invalid archive error, got %v", err)
+			}
+			if err != nil {
+				if _, statErr := os.Stat(wd); !os.IsNotExist(statErr) {
+					t.Fatalf("partial installation remains: %v", statErr)
+				}
+			}
+		})
+	}
+	files, err := os.ReadDir(filepath.Join(config.WorkDir, "gh-too-large"))
+	if err != nil || len(files) != 3 {
+		t.Fatalf("expected three persistent repo records, got %d: %v", len(files), err)
+	}
+}
+
+func TestGhInstallTimeout(t *testing.T) {
+	workDir, transport := config.WorkDir, http.DefaultTransport
+	config.WorkDir = t.TempDir()
+	t.Cleanup(func() {
+		config.WorkDir, http.DefaultTransport = workDir, transport
+	})
+	for _, phase := range []string{"request", "body", "canceled", "parent-deadline"} {
+		t.Run(phase, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) {
+					if phase == "canceled" {
+						t.Fatal("canceled install made a request")
+					}
+					if phase != "body" {
+						<-r.Context().Done()
+						return nil, r.Context().Err()
+					}
+					pr, pw := io.Pipe()
+					go func() {
+						gz := gzip.NewWriter(pw)
+						tw := tar.NewWriter(gz)
+						tw.WriteHeader(&tar.Header{Name: "repo/package.json", Mode: 0644, Size: 2})
+						tw.Write([]byte("{}"))
+						tw.Flush()
+						gz.Flush()
+						<-r.Context().Done()
+						pw.CloseWithError(r.Context().Err())
+					}()
+					return &http.Response{StatusCode: 200, Header: http.Header{}, ContentLength: -1, Body: pr}, nil
+				})
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+				if phase == "canceled" {
+					cancel()
+				} else if phase == "parent-deadline" {
+					var stop context.CancelFunc
+					ctx, stop = context.WithTimeout(ctx, time.Second)
+					defer stop()
+				}
+				wd := filepath.Join(t.TempDir(), "install")
+				start := time.Now()
+				err := ghInstallContext(ctx, wd, "owner/timeout", "main")
+				if phase == "canceled" {
+					if !errors.Is(err, context.Canceled) {
+						t.Fatalf("expected cancellation, got %v", err)
+					}
+				} else if phase == "parent-deadline" {
+					if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) != time.Second {
+						t.Fatalf("expected caller deadline, got %v after %v", err, time.Since(start))
+					}
+				} else if err == nil || err.Error() != "github: install timeout after 30 seconds" || time.Since(start) != ghInstallTimeout {
+					t.Fatalf("expected install timeout, got %v after %v", err, time.Since(start))
+				}
+				if _, err := os.Stat(wd); !os.IsNotExist(err) {
+					t.Fatalf("partial installation remains: %v", err)
+				}
+			})
+		})
+	}
+	if existsDir(filepath.Join(config.WorkDir, "gh-too-large")) {
+		t.Fatal("timed out repo was recorded as too large")
+	}
+}

server/npmrc.go

@@ -718,15 +718,19 @@ func extractPackageTarball(installDir string, pkgName string, tarball io.Reader)
 }
 
 func extractPackageTarballContext(ctx context.Context, installDir string, pkgName string, tarball io.Reader) (err error) {
-	// pkgName is joined into the extraction path below.
-	if !filepath.IsLocal(pkgName) {
-		return errors.New("invalid package name: " + pkgName)
-	}
-
 	unziped, err := gzip.NewReader(&contextReader{ctx: ctx, reader: tarball})
 	if err != nil {
 		return
 	}
+	defer unziped.Close()
+	return extractPackageTarContext(ctx, installDir, pkgName, unziped)
+}
+
+func extractPackageTarContext(ctx context.Context, installDir string, pkgName string, archive io.Reader) (err error) {
+	// pkgName is joined into the extraction path below.
+	if !filepath.IsLocal(pkgName) {
+		return errors.New("invalid package name: " + pkgName)
+	}
 
 	// Confine every write to installDir, including when a symlink is already at
 	// the destination. Lexical path checks alone cannot provide that guarantee.
@@ -740,7 +744,7 @@ func extractPackageTarballContext(ctx context.Context, installDir string, pkgNam
 	defer root.Close()
 
 	// extract tarball
-	tr := tar.NewReader(unziped)
+	tr := tar.NewReader(&contextReader{ctx: ctx, reader: archive})
 	for {
 		if err := ctx.Err(); err != nil {
 			return err