diff --git a/server/build.go b/server/build.go index d8f516b5e..4af6e8409 100644 --- a/server/build.go +++ b/server/build.go @@ -582,10 +582,9 @@ func (ctx *BuildContext) buildModule(analyzeMode bool) (meta *BuildMeta, include isSelfRef := specifier == ctx.esmPath.PkgName if !isSelfRef && strings.HasPrefix(ctx.esmPath.PkgName, "@") { _, baseName := utils.SplitByFirstByte(ctx.esmPath.PkgName[1:], '/') - specPkgName := toPackageName(specifier) - if specPkgName == baseName { - _, inDeps := pkgJson.Dependencies[specPkgName] - _, inPeerDeps := pkgJson.PeerDependencies[specPkgName] + if specifier == baseName { + _, inDeps := pkgJson.Dependencies[baseName] + _, inPeerDeps := pkgJson.PeerDependencies[baseName] if !inDeps && !inPeerDeps { isSelfRef = true } diff --git a/server/build_resolver.go b/server/build_resolver.go index 39912e573..862b31ed5 100644 --- a/server/build_resolver.go +++ b/server/build_resolver.go @@ -765,10 +765,9 @@ func (ctx *BuildContext) resolveExternalModule(specifier string, kind esbuild.Re isSelfRef := specifier == pkgJson.Name || specifier == pkgJson.PkgName if !isSelfRef && strings.HasPrefix(pkgJson.Name, "@") { _, baseName := utils.SplitByFirstByte(pkgJson.Name[1:], '/') - specPkgName := toPackageName(specifier) - if specPkgName == baseName { - _, inDeps := pkgJson.Dependencies[specPkgName] - _, inPeerDeps := pkgJson.PeerDependencies[specPkgName] + if specifier == baseName { + _, inDeps := pkgJson.Dependencies[baseName] + _, inPeerDeps := pkgJson.PeerDependencies[baseName] if !inDeps && !inPeerDeps { isSelfRef = true } @@ -784,10 +783,9 @@ func (ctx *BuildContext) resolveExternalModule(specifier string, kind esbuild.Re if ctx.pkgJson.SideEffectsFalse { sideEffects = esbuild.SideEffectsFalse } else if ctx.pkgJson.SideEffects.Len() > 0 { - sideEffects = esbuild.SideEffectsFalse entry := ctx.resolveEntry(esmPath) - if entry.main != "" && !(ctx.pkgJson.SideEffects.Has(entry.main) || ctx.pkgJson.SideEffects.Has(strings.TrimPrefix(entry.main, "./"))) { - sideEffects = esbuild.SideEffectsTrue + if entry.main != "" && !ctx.pkgJson.SideEffects.Has(entry.main) && !ctx.pkgJson.SideEffects.Has(strings.TrimPrefix(entry.main, "./")) { + sideEffects = esbuild.SideEffectsFalse } } resolvedPath = ctx.getImportPath(esmPath, ctx.getBuildArgsPrefix(false), ctx.externalAll) @@ -823,10 +821,9 @@ func (ctx *BuildContext) resolveExternalModule(specifier string, kind esbuild.Re if ctx.pkgJson.SideEffectsFalse { sideEffects = esbuild.SideEffectsFalse } else if ctx.pkgJson.SideEffects.Len() > 0 { - sideEffects = esbuild.SideEffectsFalse entry := ctx.resolveEntry(subModule) - if entry.main != "" && !(ctx.pkgJson.SideEffects.Has(entry.main) || ctx.pkgJson.SideEffects.Has(strings.TrimPrefix(entry.main, "./"))) { - sideEffects = esbuild.SideEffectsTrue + if entry.main != "" && !ctx.pkgJson.SideEffects.Has(entry.main) && !ctx.pkgJson.SideEffects.Has(strings.TrimPrefix(entry.main, "./")) { + sideEffects = esbuild.SideEffectsFalse } } if withTypeJSON { @@ -1311,8 +1308,8 @@ func (ctx *BuildContext) lexer(entry *BuildEntry) (ret *BuildMeta, cjsExports [] func matchAsteriskExport(exportName string, subModuleName string) (diff string, match bool) { if strings.ContainsRune(exportName, '*') { prefix, suffix := utils.SplitByLastByte(exportName, '*') - if strings.HasPrefix("./"+subModuleName, prefix) && strings.HasSuffix(subModuleName, suffix) { - return strings.TrimPrefix("./"+subModuleName, prefix), true + if name := "./" + subModuleName; len(name) >= len(prefix)+len(suffix) && strings.HasPrefix(name, prefix) && strings.HasSuffix(name, suffix) { + return name[len(prefix) : len(name)-len(suffix)], true } } return "", false diff --git a/server/build_resolver_test.go b/server/build_resolver_test.go new file mode 100644 index 000000000..5ac78dcf1 --- /dev/null +++ b/server/build_resolver_test.go @@ -0,0 +1,190 @@ +package server + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/esm-dev/esm.sh/internal/npm" + "github.com/esm-dev/esm.sh/internal/storage" + esbuild "github.com/ije/esbuild-internal/api" + "github.com/ije/gox/log" + "github.com/ije/gox/set" +) + +func TestResolveExternalModuleSideEffects(t *testing.T) { + for _, specifier := range []string{"example", "example/init"} { + for _, tt := range []struct { + name string + listed []string + pure bool + missing bool + keep bool + }{ + {name: "listed", listed: []string{"./init.js"}, keep: true}, + {name: "listed without dot", listed: []string{"init.js"}, keep: true}, + {name: "unlisted", listed: []string{"./other.js"}}, + {name: "unspecified", keep: true}, + {name: "false", pure: true}, + {name: "unresolved", listed: []string{"./other.js"}, missing: true, keep: true}, + } { + t.Run(specifier+"/"+tt.name, func(t *testing.T) { + wd := t.TempDir() + pkgDir := filepath.Join(wd, "node_modules", "example") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + if !tt.missing { + if err := os.WriteFile(filepath.Join(pkgDir, "init.js"), []byte("globalThis.initialized = true; export {};"), 0644); err != nil { + t.Fatal(err) + } + } + ctx := &BuildContext{ + wd: wd, target: "es2022", esmPath: EsmPath{PkgName: "example", PkgVersion: "1.0.0"}, + pkgJson: &npm.PackageJSON{ + Name: "example", Version: "1.0.0", Type: "module", Main: "./init.js", + SideEffects: *set.NewReadOnly(tt.listed...), SideEffectsFalse: tt.pure, + }, + } + resolved, sideEffects, err := ctx.resolveExternalModule(specifier, esbuild.ResolveJSImportStatement, false, false) + if err != nil { + t.Fatal(err) + } + result := esbuild.Build(esbuild.BuildOptions{ + Stdin: &esbuild.StdinOptions{Contents: fmt.Sprintf("import %q; export const value = 1;", specifier)}, + Bundle: true, Format: esbuild.FormatESModule, + Plugins: []esbuild.Plugin{{Name: "external", Setup: func(build esbuild.PluginBuild) { + build.OnResolve(esbuild.OnResolveOptions{Filter: ".*"}, func(args esbuild.OnResolveArgs) (esbuild.OnResolveResult, error) { + return esbuild.OnResolveResult{Path: resolved, External: true, SideEffects: sideEffects}, nil + }) + }}}, + }) + if len(result.Errors) != 0 { + t.Fatal(result.Errors) + } + output := string(result.OutputFiles[0].Contents) + if strings.Contains(output, resolved) != tt.keep { + t.Fatalf("import retained = %v, want %v:\n%s", !tt.keep, tt.keep, output) + } + }) + } + } +} + +func TestResolveExternalModuleScopedFork(t *testing.T) { + for _, dependency := range []string{"", "dependencies", "peerDependencies"} { + for _, subpath := range []string{"", "/tsl"} { + t.Run(dependency+subpath, func(t *testing.T) { + pkg := &npm.PackageJSON{Name: "@scope/three", Version: "1.0.0"} + want := "/@scope/three@1.0.0/es2022/" + switch dependency { + case "dependencies": + pkg.Dependencies = map[string]string{"three": "2.0.0"} + want = "/three@2.0.0/es2022/" + case "peerDependencies": + pkg.PeerDependencies = map[string]string{"three": "2.0.0"} + want = "/three@2.0.0/es2022/" + } + if subpath == "" { + want += "three.mjs" + } else { + want += "tsl.mjs" + } + ctx := &BuildContext{target: "es2022", esmPath: EsmPath{PkgName: pkg.Name, PkgVersion: pkg.Version}, pkgJson: pkg} + got, _, err := ctx.resolveExternalModule("three"+subpath, esbuild.ResolveJSImportStatement, false, false) + if err != nil || got != want { + t.Fatalf("resolved = %q, %v; want %q", got, err, want) + } + }) + } + } +} + +func TestBuildModuleScopedForkSubpath(t *testing.T) { + wd := t.TempDir() + pkgDir := filepath.Join(wd, "node_modules", "@scope", "three") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkgDir, "addon.js"), []byte(`export { value } from "three/tsl";`), 0644); err != nil { + t.Fatal(err) + } + fs, err := storage.NewFSStorage(filepath.Join(wd, "storage")) + if err != nil { + t.Fatal(err) + } + logger, err := log.New("") + if err != nil { + t.Fatal(err) + } + logger.SetOutput(io.Discard) + ctx := &BuildContext{ + wd: wd, target: "es2022", storage: fs, logger: logger, npmrc: &NpmRC{}, + esmPath: EsmPath{PkgName: "@scope/three", PkgVersion: "1.0.0", SubPath: "addon"}, + pkgJson: &npm.PackageJSON{Name: "@scope/three", Version: "1.0.0", Type: "module", Types: "./index.d.ts"}, + } + meta, _, err := ctx.buildModule(false) + if err != nil { + t.Fatal(err) + } + f, _, err := fs.Get(ctx.getSavePath()) + if err != nil { + t.Fatal(err) + } + defer f.Close() + output, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + const want = "/@scope/three@1.0.0/es2022/tsl.mjs" + if len(meta.Imports) != 1 || meta.Imports[0] != want || !strings.Contains(string(output), want) { + t.Fatalf("expected subpath import, got %v:\n%s", meta.Imports, output) + } +} + +func TestResolveEntryWildcardSuffix(t *testing.T) { + wd := t.TempDir() + pkgDir := filepath.Join(wd, "node_modules", "example", "src") + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + for _, filename := range []string{"one.js", "one.d.ts"} { + if err := os.WriteFile(filepath.Join(pkgDir, filename), []byte("export {};"), 0644); err != nil { + t.Fatal(err) + } + } + for _, conditions := range []any{ + "./src/*.js", + npm.NewJSONObject([]string{"types", "import"}, map[string]any{"types": "./src/*.d.ts", "import": "./src/*.js"}), + } { + esm := EsmPath{PkgName: "example", PkgVersion: "1.0.0", SubPath: "features/one-feature"} + ctx := &BuildContext{ + wd: wd, target: "es2022", esmPath: esm, + pkgJson: &npm.PackageJSON{ + Name: "example", Version: "1.0.0", Type: "module", + Exports: npm.NewJSONObject([]string{"./features/*-feature"}, map[string]any{"./features/*-feature": conditions}), + }, + } + if entry := ctx.resolveEntry(esm); entry.main != "./src/one.js" || entry.types != "./src/one.d.ts" || !entry.module { + t.Fatalf("unexpected entry: %+v", entry) + } + } + for _, tt := range []struct { + pattern, subpath, want string + match bool + }{ + {"./features/*", "features/one", "one", true}, + {"./features/*-feature", "features/one-feature", "one", true}, + {"./features/*-feature", "features/nested/one-feature", "nested/one", true}, + {"./features/*-feature", "features/one", "", false}, + {"./features/*-feature", "other/one-feature", "", false}, + {"./features/*features", "features", "", false}, + } { + if got, ok := matchAsteriskExport(tt.pattern, tt.subpath); got != tt.want || ok != tt.match { + t.Errorf("match(%q, %q) = %q, %v; want %q, %v", tt.pattern, tt.subpath, got, ok, tt.want, tt.match) + } + } +} diff --git a/server/config.go b/server/config.go index a219e9f95..44ca4bb5e 100644 --- a/server/config.go +++ b/server/config.go @@ -256,16 +256,12 @@ func normalizeConfig(config *Config) { config.NpmScopedRegistries = regs } if config.NpmQueryCacheTTL == 0 { - v := os.Getenv("NPM_QUERY_CACHE_TTL") - if v != "" { - i, e := strconv.Atoi(v) - if e == nil && i >= 0 { + config.NpmQueryCacheTTL = 600 + if v := os.Getenv("NPM_QUERY_CACHE_TTL"); v != "" { + if i, err := strconv.ParseUint(v, 10, 32); err == nil { config.NpmQueryCacheTTL = uint32(i) - } else { - config.NpmQueryCacheTTL = 600 } } - config.NpmQueryCacheTTL = 600 } config.Compress = !(bytes.Equal(config.CompressRaw, []byte("false")) || os.Getenv("COMPRESS") == "false") config.SourceMap = !(bytes.Equal(config.SourceMapRaw, []byte("false")) || (os.Getenv("SOURCEMAP") == "false" || os.Getenv("SOURCE_MAP") == "false")) diff --git a/server/config_test.go b/server/config_test.go index 3b46349b7..79a4ff809 100644 --- a/server/config_test.go +++ b/server/config_test.go @@ -4,6 +4,31 @@ import ( "testing" ) +func TestNpmQueryCacheTTL(t *testing.T) { + for _, test := range []struct { + env string + setting uint32 + expected uint32 + }{ + {"", 0, 600}, + {"30", 0, 30}, + {"0", 0, 0}, + {"invalid", 0, 600}, + {"-1", 0, 600}, + {"4294967296", 0, 600}, + {"30", 90, 90}, + } { + t.Run(test.env, func(t *testing.T) { + t.Setenv("NPM_QUERY_CACHE_TTL", test.env) + c := &Config{NpmQueryCacheTTL: test.setting} + normalizeConfig(c) + if c.NpmQueryCacheTTL != test.expected { + t.Fatalf("NpmQueryCacheTTL = %d, want %d", c.NpmQueryCacheTTL, test.expected) + } + }) + } +} + func TestExtractPackageName(t *testing.T) { type want struct { packageId string diff --git a/server/middleware.go b/server/middleware.go index 00dad8b53..3a931e82d 100644 --- a/server/middleware.go +++ b/server/middleware.go @@ -3,6 +3,7 @@ package server import ( "compress/gzip" "io" + "mime" "net" "net/http" "runtime/debug" @@ -95,16 +96,44 @@ func (w *loggedResponseWriter) Unwrap() http.ResponseWriter { // or gzip if the client accepts it. func withCompress(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + brQuality, gzipQuality, wildcardQuality, identityQuality := -1.0, -1.0, 0.0, 0.0 + for _, value := range r.Header.Values("Accept-Encoding") { + for part := range strings.SplitSeq(value, ",") { + coding, params, err := mime.ParseMediaType(part) + if err != nil { + continue + } + quality := 1.0 + if q, ok := params["q"]; ok { + quality, err = strconv.ParseFloat(q, 64) + if err != nil || quality < 0 || quality > 1 { + quality = 0 + } + } + switch coding { + case "br": + brQuality = quality + case "gzip": + gzipQuality = quality + case "*": + wildcardQuality = quality + case "identity": + identityQuality = quality + } + } + } + if brQuality < 0 { + brQuality = wildcardQuality + } + if gzipQuality < 0 { + gzipQuality = wildcardQuality + } var encoding string - if acceptEncoding := r.Header.Get("Accept-Encoding"); strings.Contains(acceptEncoding, "br") { + if brQuality > 0 && brQuality >= gzipQuality && brQuality >= identityQuality { encoding = "br" - } else if strings.Contains(acceptEncoding, "gzip") { + } else if gzipQuality > 0 && gzipQuality >= identityQuality { encoding = "gzip" } - if encoding == "" { - next.ServeHTTP(w, r) - return - } wr := &compressResponseWriter{ResponseWriter: w, encoding: encoding} defer wr.Close() next.ServeHTTP(wr, r) @@ -135,12 +164,14 @@ func (w *compressResponseWriter) WriteHeader(code int) { } if size < 0 || size >= compressMinSize { appendVaryHeader(h, "Accept-Encoding") - h.Set("Content-Encoding", w.encoding) - h.Del("Content-Length") - if w.encoding == "br" { - w.zWriter = brotli.NewWriterLevel(w.ResponseWriter, brotli.BestSpeed) - } else { - w.zWriter, _ = gzip.NewWriterLevel(w.ResponseWriter, gzip.BestSpeed) + if w.encoding != "" { + h.Set("Content-Encoding", w.encoding) + h.Del("Content-Length") + if w.encoding == "br" { + w.zWriter = brotli.NewWriterLevel(w.ResponseWriter, brotli.BestSpeed) + } else { + w.zWriter, _ = gzip.NewWriterLevel(w.ResponseWriter, gzip.BestSpeed) + } } } } diff --git a/server/middleware_test.go b/server/middleware_test.go new file mode 100644 index 000000000..671b86957 --- /dev/null +++ b/server/middleware_test.go @@ -0,0 +1,74 @@ +package server + +import ( + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/andybalholm/brotli" +) + +func TestCompressNegotiation(t *testing.T) { + body := strings.Repeat("export const value = 1;\n", 100) + handler := withCompress(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/javascript") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.Header().Set("Vary", "Origin") + io.WriteString(w, body) + })) + for _, test := range []struct { + accept []string + encoding string + }{ + {nil, ""}, + {[]string{"gzip"}, "gzip"}, + {[]string{"br, gzip"}, "br"}, + {[]string{"gzip, br;q=0"}, "gzip"}, + {[]string{"gzip;q=0, identity;q=1"}, ""}, + {[]string{"br;q=0, gzip;q=0"}, ""}, + {[]string{"br;q=0.2, gzip;q=0.8"}, "gzip"}, + {[]string{"gzip;q=0.5, identity;q=1"}, ""}, + {[]string{"*"}, "br"}, + {[]string{"br;q=0, *;q=1"}, "gzip"}, + {[]string{"*;q=1, br;q=0, gzip;q=0"}, ""}, + {[]string{"*;q=0, gzip"}, "gzip"}, + {[]string{"xbr, gzipx"}, ""}, + {[]string{"br;q=invalid, gzip;q=2"}, ""}, + {[]string{"gzip", "br;q=0"}, "gzip"}, + } { + t.Run(strings.Join(test.accept, "/"), func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/module.js", nil) + for _, value := range test.accept { + r.Header.Add("Accept-Encoding", value) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if got := w.Header().Get("Content-Encoding"); got != test.encoding { + t.Fatalf("Content-Encoding = %q, want %q", got, test.encoding) + } + if got := w.Header().Get("Vary"); got != "Origin, Accept-Encoding" { + t.Errorf("Vary = %q", got) + } + var reader io.Reader = w.Body + switch test.encoding { + case "gzip": + gz, err := gzip.NewReader(w.Body) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + reader = gz + case "br": + reader = brotli.NewReader(w.Body) + } + data, err := io.ReadAll(reader) + if err != nil || string(data) != body { + t.Fatalf("unexpected response body: %v", err) + } + }) + } +} diff --git a/server/npmrc.go b/server/npmrc.go index f2723191c..f6f7b0094 100644 --- a/server/npmrc.go +++ b/server/npmrc.go @@ -202,7 +202,7 @@ CHECK: return distVersion, nil } } else { - if version == "lastest" { + if version == "latest" { return "", fmt.Errorf("version %s not found", version) } c, err := semver.NewConstraint(version) @@ -652,7 +652,8 @@ func fetchPackageTarballContext(ctx context.Context, reg *NpmRegistry, installDi return false } - if reg.isRateLimited() && reg.BackupRegistry != "" && strings.HasPrefix(tarballUrlStr, reg.Registry) { + useBackup := reg.isRateLimited() && reg.BackupRegistry != "" && strings.HasPrefix(tarballUrlStr, reg.Registry) + if useBackup { var backupUrl *url.URL backupUrl, err = url.Parse(reg.BackupRegistry) if err != nil { @@ -706,7 +707,7 @@ RETRY: return } - if res.StatusCode == 429 && reg.isRateLimited() && reg.BackupRegistry != "" && strings.HasPrefix(tarballUrlStr, reg.Registry) { + if res.StatusCode == 429 && !useBackup && reg.BackupRegistry != "" && strings.HasPrefix(tarballUrlStr, reg.Registry) { var backupUrl *url.URL backupUrl, err = url.Parse(reg.BackupRegistry) if err != nil { @@ -716,6 +717,7 @@ RETRY: backupUrl.RawQuery = tarballUrl.RawQuery tarballUrl = backupUrl tarballUrlStr = backupUrl.String() + useBackup = true reg.hitRateLimit() goto RETRY } diff --git a/server/npmrc_test.go b/server/npmrc_test.go index 063f89362..f3c691232 100644 --- a/server/npmrc_test.go +++ b/server/npmrc_test.go @@ -13,12 +13,47 @@ import ( "net/url" "os" "path/filepath" + "slices" "testing" "time" "github.com/esm-dev/esm.sh/internal/npm" ) +func TestResolveSemverVersion(t *testing.T) { + for _, test := range []struct { + name string + version string + tags map[string]string + want string + }{ + {"missing latest", "latest", nil, ""}, + {"unknown tag without latest", "unknown", nil, ""}, + {"dangling latest", "latest", map[string]string{"latest": "3.0.0"}, ""}, + {"latest", "latest", map[string]string{"latest": "1.2.0"}, "1.2.0"}, + {"unknown tag falls back to latest", "unknown", map[string]string{"latest": "1.2.0"}, "1.2.0"}, + {"named tag", "next", map[string]string{"next": "2.0.0-beta.1"}, "2.0.0-beta.1"}, + {"highest matching version", "^1", nil, "1.10.0"}, + {"stable wildcard", "*", nil, "1.10.0"}, + {"no matching version", "^3", nil, ""}, + } { + t.Run(test.name, func(t *testing.T) { + metadata := &npm.PackageMetadata{ + DistTags: test.tags, + Versions: map[string]npm.PackageJSONRaw{ + "1.2.0": {Version: "1.2.0"}, + "1.10.0": {Version: "1.10.0"}, + "2.0.0-beta.1": {Version: "2.0.0-beta.1"}, + }, + } + got, err := resolveSemverVersion(metadata, test.version) + if got != test.want || (err != nil) != (test.want == "") { + t.Fatalf("resolveSemverVersion(%q) = %q, %v; want %q", test.version, got, err, test.want) + } + }) + } +} + func TestInvalidateDistTagCacheIfNewer(t *testing.T) { tests := []struct { request string @@ -128,6 +163,84 @@ func TestFetchPackageTarballAuthorization(t *testing.T) { } } +func TestFetchPackageTarballBackup(t *testing.T) { + var tarball bytes.Buffer + gw := gzip.NewWriter(&tarball) + tw := tar.NewWriter(gw) + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gw.Close(); err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + primaryStatus int + backupStatus int + rateLimited bool + redirectBackup bool + wantRequests []string + wantErr bool + }{ + {"primary succeeds", 200, 200, false, false, []string{"primary Bearer secret"}, false}, + {"first rate limit", 429, 200, false, false, []string{"primary Bearer secret", "backup Bearer secret"}, false}, + {"already rate limited", 429, 200, true, false, []string{"backup Bearer secret"}, false}, + {"backup rate limited", 429, 429, false, false, []string{"primary Bearer secret", "backup Bearer secret"}, true}, + {"backup redirects to untrusted origin", 429, 200, false, true, []string{"primary Bearer secret", "backup Bearer secret", "external "}, false}, + } { + t.Run(test.name, func(t *testing.T) { + requests := make(chan string, 16) + external := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- "external " + r.Header.Get("Authorization") + _, _ = w.Write(tarball.Bytes()) + })) + defer external.Close() + backup := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- "backup " + r.Header.Get("Authorization") + if r.URL.RequestURI() != "/test-package.tgz?download=1" { + t.Errorf("backup request lost path or query: %s", r.URL) + } + if test.redirectBackup { + http.Redirect(w, r, external.URL+"/test-package.tgz", http.StatusFound) + return + } + w.WriteHeader(test.backupStatus) + _, _ = w.Write(tarball.Bytes()) + })) + defer backup.Close() + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- "primary " + r.Header.Get("Authorization") + w.WriteHeader(test.primaryStatus) + _, _ = w.Write(tarball.Bytes()) + })) + defer primary.Close() + reg := &NpmRegistry{NpmRegistryConfig: NpmRegistryConfig{ + Registry: primary.URL + "/", BackupRegistry: backup.URL + "/", Token: "secret", + }} + if test.rateLimited { + reg.rateLimited.Store(1) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := fetchPackageTarballContext(ctx, reg, t.TempDir(), "test-package", primary.URL+"/test-package.tgz?download=1") + if (err != nil) != test.wantErr { + t.Fatalf("fetchPackageTarballContext() = %v; want error: %v", err, test.wantErr) + } + if test.primaryStatus == 429 && !reg.isRateLimited() { + t.Fatal("registry rate limit was not recorded") + } + got := make([]string, 0, len(requests)) + for len(requests) > 0 { + got = append(got, <-requests) + } + if !slices.Equal(got, test.wantRequests) { + t.Fatalf("requests = %q; want %q", got, test.wantRequests) + } + }) + } +} + func TestExtractPackageTarball(t *testing.T) { b := make([]byte, 16) rand.Read(b) diff --git a/server/path.go b/server/path.go index ed57eeeee..b5cd884b3 100644 --- a/server/path.go +++ b/server/path.go @@ -6,7 +6,6 @@ import ( "net/http" "net/url" "regexp" - "sort" "strings" "time" @@ -324,23 +323,17 @@ func resolveGhPackageVersion(esm EsmPath) (version string, err error) { } // try to find the 'semver' tag if semv, erro := semver.NewConstraint(strings.TrimPrefix(esm.PkgVersion, "semver:")); erro == nil { - semtags := make([]*semver.Version, len(refs)) - i := 0 + var latest *semver.Version for _, ref := range refs { if after, ok := strings.CutPrefix(ref.Ref, "refs/tags/"); ok { v, e := semver.NewVersion(after) - if e == nil && semv.Check(v) { - semtags[i] = v - i++ + if e == nil && semv.Check(v) && (latest == nil || v.GreaterThan(latest)) { + latest = v + version = ref.Sha[:7] } } } - if i > 0 { - semtags = semtags[:i] - if i > 1 { - sort.Sort(semver.Collection(semtags)) - } - version = semtags[i-1].String() + if latest != nil { return } } diff --git a/server/path_test.go b/server/path_test.go index 075ee6a3d..1456cdbd9 100644 --- a/server/path_test.go +++ b/server/path_test.go @@ -3,8 +3,52 @@ package server import ( "net/http" "testing" + "time" ) +func TestParseEsmPathGithubVersion(t *testing.T) { + const repo = "path-test/semver-tags" + key := "git ls-remote https://github.com/" + repo + setCacheItem(key, []GitRef{ + {Ref: "HEAD", Sha: "abcdef1234567890"}, + {Ref: "refs/heads/main", Sha: "abcdef1234567890"}, + {Ref: "refs/tags/v1.10.0", Sha: "1111111234567890"}, + {Ref: "refs/tags/1.2.0", Sha: "2222222234567890"}, + {Ref: "refs/tags/v2.0.0-beta.1", Sha: "3333333234567890"}, + {Ref: "refs/tags/invalid", Sha: "4444444234567890"}, + }, time.Minute) + t.Cleanup(func() { deleteCacheItem(key) }) + for _, test := range []struct { + version string + want string + exact bool + }{ + {"", "abcdef1", false}, + {"main", "abcdef1", false}, + {"^1", "1111111", false}, + {"semver:^1", "1111111", false}, + {"~1.2", "2222222", false}, + {"*", "1111111", false}, + {"v1.10.0", "v1.10.0", true}, + {"abcdef1", "abcdef1", true}, + {"^3", "", false}, + } { + t.Run(test.version, func(t *testing.T) { + t.Cleanup(func() { deleteCacheItem("gh/" + repo + "@" + test.version) }) + esm, _, exact, _, _, err := parseEsmPath(nil, "/gh/"+repo+"@"+test.version) + if test.want == "" { + if err == nil { + t.Fatal("expected an unmatched version error") + } + return + } + if err != nil || esm.PkgVersion != test.want || exact != test.exact { + t.Fatalf("version %q = %q (exact %v), %v; want %q (exact %v)", test.version, esm.PkgVersion, exact, err, test.want, test.exact) + } + }) + } +} + func TestPrCommitFromHeader(t *testing.T) { tests := []struct { name string diff --git a/server/router.go b/server/router.go index 86e071783..584ba8c14 100644 --- a/server/router.go +++ b/server/router.go @@ -127,6 +127,7 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { h.Write([]byte(options.JSXImportSource)) h.Write([]byte(options.SourceMap)) fmt.Fprintf(h, "%v", options.Minify) + h.Write([]byte(options.Filename)) hash := hex.EncodeToString(h.Sum(nil)) savePath := normalizeSavePath(fmt.Sprintf("modules/transform/%s.mjs", hash)) @@ -910,14 +911,15 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { // return css file as a `CSSStyleSheet` object when `?module` query is present if pathKind == RawFile && strings.HasSuffix(esmPath.SubPath, ".css") && query.Has("module") { - if esmPath.GhPrefix { - if _, err := npmrc.installPackageContext(r.Context(), esmPath.Package()); err != nil { + filename := path.Join(npmrc.StoreDir(), esmPath.PackageId(), "node_modules", esmPath.PkgName, esmPath.SubPath) + css, err := os.ReadFile(filename) + if os.IsNotExist(err) { + if _, err = npmrc.installPackageContext(r.Context(), esmPath.Package()); err != nil { writeStatus(w, 500, err.Error()) return } + css, err = os.ReadFile(filename) } - filename := path.Join(npmrc.StoreDir(), esmPath.PackageId(), "node_modules", esmPath.PkgName, esmPath.SubPath) - css, err := os.ReadFile(filename) if err != nil { writeStatus(w, 500, err.Error()) return @@ -1736,9 +1738,14 @@ func esmRouter(esmStorage storage.Storage, logger *log.Logger) http.Handler { if buildMeta.ExportDefault && (len(exports) == 0 || slices.Contains(exports, "default")) { fmt.Fprintf(buf, "export { default } from \"%s\";\n", esmPath) } - if buildMeta.CJS && len(exports) > 0 { - fmt.Fprintf(buf, "import _ from \"%s\";\n", esmPath) - fmt.Fprintf(buf, "export const { %s } = _;\n", strings.Join(exports, ", ")) + if buildMeta.CJS { + if i := slices.Index(exports, "default"); i >= 0 { + exports = slices.Delete(exports, i, i+1) + } + if len(exports) > 0 { + fmt.Fprintf(buf, "import _ from \"%s\";\n", esmPath) + fmt.Fprintf(buf, "export const { %s } = _;\n", strings.Join(exports, ", ")) + } } header.Set("X-ESM-Path", esmPath) if noDts := query.Has("no-dts") || query.Has("no-check"); !noDts && buildMeta.Dts != "" { diff --git a/server/router_test.go b/server/router_test.go index bb8275de5..38b09338f 100644 --- a/server/router_test.go +++ b/server/router_test.go @@ -1,6 +1,12 @@ package server import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -10,10 +16,152 @@ import ( "testing" "time" + "github.com/esm-dev/esm.sh/internal/npm" "github.com/esm-dev/esm.sh/internal/storage" + esbuild "github.com/ije/esbuild-internal/api" "github.com/ije/gox/log" ) +func TestRouterModuleResponses(t *testing.T) { + previousConfig, previousNpmRC, transport := config, defaultNpmRC, http.DefaultTransport + testConfig := *config + testConfig.WorkDir = t.TempDir() + config, defaultNpmRC = &testConfig, nil + t.Cleanup(func() { + config, defaultNpmRC, http.DefaultTransport = previousConfig, previousNpmRC, transport + }) + fs, err := storage.NewFSStorage(filepath.Join(config.WorkDir, "storage")) + if err != nil { + t.Fatal(err) + } + logger := new(log.Logger) + logger.SetOutput(io.Discard) + handler := esmRouter(fs, logger) + + t.Run("css module installation", func(t *testing.T) { + var archive bytes.Buffer + gz := gzip.NewWriter(&archive) + tw := tar.NewWriter(gz) + for name, content := range map[string]string{ + "package.json": `{"name":"css-module-test","version":"1.0.0"}`, + "style.css": "body { color: red }", + } { + if err := tw.WriteHeader(&tar.Header{Name: "package/" + name, Mode: 0644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, content); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + setCacheItem("npm:css-module-test@1.0.0", &npm.PackageJSON{ + Name: "css-module-test", Version: "1.0.0", + Dist: npm.NpmPackageDist{Tarball: "https://registry.example/css-module-test.tgz"}, + }, time.Minute) + defer deleteCacheItem("npm:css-module-test@1.0.0") + downloads := 0 + http.DefaultTransport = ghTestTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.String() != "https://registry.example/css-module-test.tgz" { + t.Fatalf("unexpected request: %s", r.URL) + } + downloads++ + return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(archive.Bytes()))}, nil + }) + for _, state := range []string{"cold", "warm", "evicted"} { + if state == "evicted" { + if err := os.RemoveAll(filepath.Join(config.WorkDir, "npm", "css-module-test@1.0.0")); err != nil { + t.Fatal(err) + } + } + res := httptest.NewRecorder() + handler.ServeHTTP(res, httptest.NewRequest("GET", "http://localhost/css-module-test@1.0.0/style.css?module", nil)) + if res.Code != 200 || res.Header().Get("Content-Type") != ctJavaScript || !strings.Contains(res.Body.String(), `stylesheet.replaceSync("body{color:red}")`) { + t.Fatalf("%s: HTTP %d: %s", state, res.Code, res.Body.String()) + } + wantDownloads := 1 + if state == "evicted" { + wantDownloads = 2 + } + if downloads != wantDownloads { + t.Fatalf("%s: got %d downloads, want %d", state, downloads, wantDownloads) + } + } + }) + + t.Run("commonjs default export", func(t *testing.T) { + build := &BuildContext{esmPath: EsmPath{PkgName: "cjs-exports-test", PkgVersion: "1.0.0"}, target: "es2022"} + defer cacheLRU.Remove(build.Path()) + if err := NewBuildMetaDB(fs).Put(build.Path(), encodeBuildMeta(&BuildMeta{CJS: true, ExportDefault: true})); err != nil { + t.Fatal(err) + } + for _, exports := range []string{"default", "answer,default", "answer"} { + res := httptest.NewRecorder() + handler.ServeHTTP(res, httptest.NewRequest("GET", "http://localhost/cjs-exports-test@1.0.0?target=es2022&exports="+exports, nil)) + if res.Code != 200 { + t.Fatalf("HTTP %d: %s", res.Code, res.Body.String()) + } + code := res.Body.String() + parsed := esbuild.Transform(code, esbuild.TransformOptions{Loader: esbuild.LoaderJS}) + if len(parsed.Errors) > 0 { + t.Fatalf("exports=%s: invalid JavaScript: %v\n%s", exports, parsed.Errors, code) + } + if strings.Contains(code, "export { default }") != strings.Contains(exports, "default") || strings.Contains(code, "export const { answer }") != strings.Contains(exports, "answer") { + t.Fatalf("exports=%s: incorrect exports:\n%s", exports, code) + } + } + }) + + t.Run("transform filename cache", func(t *testing.T) { + options := TransformOptions{Lang: "ts", Code: "export const answer: number = 42;", Target: "esnext", ImportMapRaw: json.RawMessage(`{}`), SourceMap: "external"} + for _, filename := range []string{"", "first.ts", "second.ts", "first.ts"} { + options.Filename = filename + body, err := json.Marshal(options) + if err != nil { + t.Fatal(err) + } + res := httptest.NewRecorder() + handler.ServeHTTP(res, httptest.NewRequest("POST", "http://localhost/transform", bytes.NewReader(body))) + if res.Code != 200 { + t.Fatalf("HTTP %d: %s", res.Code, res.Body.String()) + } + var output TransformOutput + if err := json.Unmarshal(res.Body.Bytes(), &output); err != nil { + t.Fatal(err) + } + var sourceMap struct{ Sources []string } + if err := json.Unmarshal([]byte(output.Map), &sourceMap); err != nil { + t.Fatal(err) + } + wantSource := filename + if wantSource == "" { + wantSource = "source.ts" + // /tsx computes this hash before calling the transform API. + hash := sha1.Sum([]byte(options.Lang + options.Code + options.Target + string(options.ImportMapRaw) + options.SourceMap + "false")) + if !strings.Contains(output.Code, fmt.Sprintf("sourceMappingURL=+%x.mjs.map", hash)) { + t.Fatal("transform without a filename changed its cache URL") + } + } + if len(sourceMap.Sources) != 1 || sourceMap.Sources[0] != wantSource { + t.Fatalf("filename=%q: unexpected source map: %s", filename, output.Map) + } + _, mapURL, ok := strings.Cut(output.Code, "//# sourceMappingURL=") + if !ok { + t.Fatal("missing source map URL") + } + cached := httptest.NewRecorder() + handler.ServeHTTP(cached, httptest.NewRequest("GET", "http://localhost/"+mapURL, nil)) + if cached.Code != 200 || cached.Body.String() != output.Map { + t.Fatalf("filename=%q: cached source map does not match the transform", filename) + } + } + }) +} + func TestCSSEntryRedirectURL(t *testing.T) { origin := "https://esm.sh" esmPath := EsmPath{ diff --git a/server/server.go b/server/server.go index f344fb824..cb7db3e08 100644 --- a/server/server.go +++ b/server/server.go @@ -196,7 +196,7 @@ func customLandingPage(options *LandingPageOptions, next http.Handler) http.Hand h := w.Header() etag := res.Header.Get("Etag") if etag != "" { - if r.Header.Get("If-None-Match") == etag { + if res.StatusCode == http.StatusOK && r.Header.Get("If-None-Match") == etag { w.WriteHeader(http.StatusNotModified) return } @@ -205,10 +205,10 @@ func customLandingPage(options *LandingPageOptions, next http.Handler) http.Hand lastModified := res.Header.Get("Last-Modified") if lastModified != "" { v := r.Header.Get("If-Modified-Since") - if v != "" { + if res.StatusCode == http.StatusOK && v != "" { timeIfModifiedSince, e1 := time.Parse(http.TimeFormat, v) timeLastModified, e2 := time.Parse(http.TimeFormat, lastModified) - if e1 == nil && e2 == nil && !timeIfModifiedSince.After(timeLastModified) { + if e1 == nil && e2 == nil && !timeLastModified.After(timeIfModifiedSince) { w.WriteHeader(http.StatusNotModified) return } @@ -222,6 +222,7 @@ func customLandingPage(options *LandingPageOptions, next http.Handler) http.Hand } h.Set("Cache-Control", cacheControl) h.Set("Content-Type", res.Header.Get("Content-Type")) + w.WriteHeader(res.StatusCode) io.Copy(w, res.Body) }) } diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 000000000..c3d157af4 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestCustomLandingPageResponse(t *testing.T) { + modified := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + for _, test := range []struct { + name string + upstreamCode int + since string + etag string + wantCode int + }{ + {"unconditional", 200, "", "", 200}, + {"modified", 200, modified.Add(-time.Hour).Format(http.TimeFormat), "", 200}, + {"unchanged", 200, modified.Format(http.TimeFormat), "", 304}, + {"future", 200, modified.Add(time.Hour).Format(http.TimeFormat), "", 304}, + {"invalid date", 200, "invalid", "", 200}, + {"matching etag", 200, "", `"version"`, 304}, + {"missing asset", 404, "", "", 404}, + {"error with matching date", 503, modified.Format(http.TimeFormat), "", 503}, + {"error with matching etag", 503, "", `"version"`, 503}, + } { + t.Run(test.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Header().Set("Last-Modified", modified.Format(http.TimeFormat)) + w.Header().Set("Etag", test.etag) + w.WriteHeader(test.upstreamCode) + io.WriteString(w, "landing page") + })) + defer upstream.Close() + handler := customLandingPage(&LandingPageOptions{Origin: upstream.URL, Assets: []string{"/asset"}}, http.NotFoundHandler()) + r := httptest.NewRequest(http.MethodGet, "/asset", nil) + r.Header.Set("If-Modified-Since", test.since) + r.Header.Set("If-None-Match", test.etag) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Code != test.wantCode { + t.Fatalf("status = %d, want %d", w.Code, test.wantCode) + } + wantBody := "landing page" + if test.wantCode == http.StatusNotModified { + wantBody = "" + } + if w.Body.String() != wantBody { + t.Fatalf("body = %q, want %q", w.Body.String(), wantBody) + } + }) + } +}