diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 6880ce1e9..6ef32799b 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -6,7 +6,7 @@ runs: steps: - uses: actions/setup-go@v5 with: - go-version: 1.25.x + go-version: 1.26.x cache: true check-latest: true - - uses: ko-build/setup-ko@v0.9 + - uses: ko-build/setup-ko@v0.10 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 61cb5bc8a..f47aeb851 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,8 +14,8 @@ jobs: fail-fast: false matrix: kubernetes: - - v1.33.7 - - v1.35.0 + - v1.34.11 + - v1.37.0 max-parallel: 2 runs-on: ubuntu-latest @@ -28,14 +28,14 @@ jobs: - uses: helm/kind-action@v1 with: - version: v0.31.0 + version: v0.33.0 node_image: kindest/node:${{ matrix.kubernetes }} cluster_name: kind config: test/kind/config.yaml wait: 120s - uses: shipwright-io/setup@v1 with: - tekton-version: v1.9.1 + tekton-version: v1.15.1 shipwright-ref: main cli-ref: _ignore diff --git a/go.mod b/go.mod index 66e1c82f4..4d44a0467 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,28 @@ module github.com/shipwright-io/cli -go 1.26.3 +go 1.26.4 require ( - github.com/google/go-containerregistry v0.21.5 - github.com/onsi/gomega v1.42.0 + github.com/google/go-containerregistry v0.22.1 + github.com/onsi/gomega v1.43.0 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 - github.com/schollz/progressbar/v3 v3.19.0 - github.com/shipwright-io/build v0.20.2 + github.com/schollz/progressbar/v3 v3.19.1 + github.com/shipwright-io/build v0.21.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - k8s.io/api v0.36.1 - k8s.io/apimachinery v0.36.1 - k8s.io/cli-runtime v0.36.1 - k8s.io/client-go v0.36.1 + k8s.io/api v0.36.4 + k8s.io/apimachinery v0.36.4 + k8s.io/cli-runtime v0.36.4 + k8s.io/client-go v0.36.4 k8s.io/klog/v2 v2.140.0 - k8s.io/kubectl v0.36.1 + k8s.io/kubectl v0.36.4 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/yaml v1.6.0 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.3 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -31,11 +31,10 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/docker/cli v29.5.3+incompatible // indirect - github.com/docker/docker-credential-helpers v0.9.5 // indirect + github.com/docker/cli v29.7.2+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.8 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect @@ -44,8 +43,8 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect - github.com/go-git/go-git/v5 v5.19.1 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect @@ -63,19 +62,18 @@ require ( github.com/go-openapi/swag/typeutils v0.25.4 // indirect github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.28.0 // indirect + github.com/google/cel-go v0.30.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/moby/term v0.5.0 // indirect @@ -87,61 +85,60 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect - github.com/prometheus/procfs v0.20.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/tektoncd/pipeline v1.12.0 // indirect - github.com/vbatts/tar-split v0.12.2 // indirect + github.com/tektoncd/pipeline v1.15.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.65.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.66.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.14.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260727163830-6c54dddc4772 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - k8s.io/component-base v0.36.1 // indirect - k8s.io/component-helpers v0.36.1 // indirect + k8s.io/component-base v0.36.4 // indirect + k8s.io/component-helpers v0.36.4 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect - k8s.io/streaming v0.36.1 // indirect - knative.dev/pkg v0.0.0-20260318013857-98d5a706d4fd // indirect + k8s.io/streaming v0.36.4 // indirect + knative.dev/pkg v0.0.0-20260622140654-39ebae2ee2dc // indirect sigs.k8s.io/controller-runtime v0.24.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.21.1 // indirect sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect ) diff --git a/go.sum b/go.sum index d2b8fafbc..2178a1796 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -28,8 +28,6 @@ github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7m github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/cloudevents/sdk-go/v2 v2.16.2 h1:ZYDFrYke4FD+jM8TZTJJO6JhKHzOQl2oqpFK1D+NnQM= github.com/cloudevents/sdk-go/v2 v2.16.2/go.mod h1:laOcGImm4nVJEU+PHnUrKL56CKmRL65RlQF0kRmW/kg= -github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= -github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -39,10 +37,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= -github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= -github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= +github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= +github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q= +github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= @@ -61,11 +59,11 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= -github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= -github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -110,14 +108,14 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= -github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/cel-go v0.30.0 h1:ll54AkzKunWkBn9wSoiUXbFZXYZTkdJGNXTBXUoolGo= +github.com/google/cel-go v0.30.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.5 h1:KTJG9Pn/jC0VdZR6ctV3/jcN+q6/Iqlx0sTVz3ywZlM= -github.com/google/go-containerregistry v0.21.5/go.mod h1:ySvMuiWg+dOsRW0Hw8GYwfMwBlNRTmpYBFJPlkco5zU= +github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84= +github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= @@ -125,8 +123,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -137,8 +135,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -153,8 +151,6 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= @@ -171,10 +167,10 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.31.0 h1:GtuJos5DFUV9EerYJo8RhYxosYNGvOdDE5haKq6Grfs= -github.com/onsi/ginkgo/v2 v2.31.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= -github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU= +github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -186,16 +182,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -204,14 +200,12 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= -github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/schollz/progressbar/v3 v3.19.1 h1:iv8BgwOvdML/S3p84uBpy/IMigv4U9594vPZYa2EdrU= +github.com/schollz/progressbar/v3 v3.19.1/go.mod h1:LFL7jqimKxfhero4K1eCkUr/6R39AgQeiPCJtlTWIW8= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shipwright-io/build v0.20.0 h1:gwn7ch4NSNiilMhjK5TosAYbQl6zXS+N77LfaDYzB6Q= -github.com/shipwright-io/build v0.20.0/go.mod h1:4yGXARV6zi7BJCSaLGjBjFXtwCUqQvm1xfDN9g9dQPI= -github.com/shipwright-io/build v0.20.2 h1:6RS6vhUI4j5aQOin10K3UFDcTVPCEsHeCg+QZFZqK2w= -github.com/shipwright-io/build v0.20.2/go.mod h1:4yGXARV6zi7BJCSaLGjBjFXtwCUqQvm1xfDN9g9dQPI= +github.com/shipwright-io/build v0.21.0 h1:2z+CJJLqrCKO0YIz8OWIzzBCHp41fbvP2KGc2qg5B8Y= +github.com/shipwright-io/build v0.21.0/go.mod h1:10w/ql4Ukbg/ErXsxRR6Lb8IYXb7P39vpChNcOnXMNY= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -227,44 +221,44 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tektoncd/pipeline v1.12.0 h1:/a2ENIZYLwvvApWU8KHL5CWCj/fNjiGU/aTUu3up2YQ= -github.com/tektoncd/pipeline v1.12.0/go.mod h1:FaS4+AjlUmMOqkzLtdjaY40iLGbCiLkal/BsJ102FLI= +github.com/tektoncd/pipeline v1.15.1 h1:eCGlMonJr7OhiLaaykO/+qXZv15eijswriT7JrLW9aQ= +github.com/tektoncd/pipeline v1.15.1/go.mod h1:t2/vjjHPODK3hvx1OFgWVyo9Va/gGE1gR3o/beAsXKg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= -github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= -github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0/go.mod h1:2lmweYCiHYpEjQ/lSJBYhj9jP1zvCvQW4BqL9dnT7FQ= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= -go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18BqlOuz2PWEvs8yM7Q6XE= -go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252ccGzkEqLc2aSXbQIALYQ= +go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D/gAmznQTd4dxxMA93aKbcpKqiTCs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -273,41 +267,42 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/api v0.0.0-20260727163830-6c54dddc4772 h1:4namukbyF7JY83aWHQwi9J5ugNTnDReLJ9ZcpqOpRB4= +google.golang.org/genproto/googleapis/api v0.0.0-20260727163830-6c54dddc4772/go.mod h1:1brfde68Npq6+WA75c1EHWPijZEG1kMus61ygPZfn4A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -322,30 +317,30 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= -k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= -k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= -k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= -k8s.io/cli-runtime v0.36.1 h1:yuC/BGnnj1YYPh6D1P+pZnzinCs6DvMq86yAeNqoqzM= -k8s.io/cli-runtime v0.36.1/go.mod h1:ZQWHGt8xAF7KnviB79vX0lYNyUUqKIpU+LQg7exuFAw= -k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= -k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= -k8s.io/component-base v0.36.1 h1:iG6GsELftXqTNG9HG6kiVjatSgAw1sf5pJ6R5a6N0kA= -k8s.io/component-base v0.36.1/go.mod h1:nf9XPlntRdqO6WMeEWAA5F93Y4ICZQdeT9GeqLDB3JI= -k8s.io/component-helpers v0.36.1 h1:BTrr5fzNSm8TkQfXrKT3N9ioWwiC4n2FTIwGTUo/ccg= -k8s.io/component-helpers v0.36.1/go.mod h1:s38HnzKQRurbUnhI5IV8GwyL/a3lVuNCYZMTd+rITMM= +k8s.io/api v0.36.4 h1:RxrvqCL6vgH5/+UnTeu1IIFqYmGfy0hnyrod1rn35Oo= +k8s.io/api v0.36.4/go.mod h1:S2B3orCFBDhrgyWbLeuKcT2QdHIpQesBkCYSlWtwUOw= +k8s.io/apimachinery v0.36.4 h1:PT2UzkupGuAx/+xT5XjiMJ1WGpY3fn9/hdAvjweRet4= +k8s.io/apimachinery v0.36.4/go.mod h1:p2I2dipt7JHG+quVwQ1d02d28O4GdDi77RByQ13MTpk= +k8s.io/cli-runtime v0.36.4 h1:OHvManCwP1k9GiC5tXRFxHhzZIQQFCsrHlt7OspKo3w= +k8s.io/cli-runtime v0.36.4/go.mod h1:qQSj2FJgQos6GHpS/ge7wTdQMZm9XFWlesWgV6h7qZY= +k8s.io/client-go v0.36.4 h1:MDvfDNvMSt0Br94SK8neviVlwL9qifw9B26hJCpD1K0= +k8s.io/client-go v0.36.4/go.mod h1:pNK4WKELbwlEDvtbE8l22lEZL5THYF61H5EealokZmA= +k8s.io/component-base v0.36.4 h1:tz75yC2xgq3kd7vPdBtR8do5iMx0OHf6Zd1kuaxDB84= +k8s.io/component-base v0.36.4/go.mod h1:DCwb306U8ou89NNAp45Csuy8ok+1rp1ELDVPhzN5AWc= +k8s.io/component-helpers v0.36.4 h1:B5+SnptAlwTVJO0MW1JLsZVp4KevlADRg0CX5YFzjbU= +k8s.io/component-helpers v0.36.4/go.mod h1:RECUDkRdVuxcNOAl9/EK7lmy1TIjoWkBHs6f0uOyoCs= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= -k8s.io/kubectl v0.36.1 h1:96HqS9twIdHM0MlJLTwbo14b9kUKPkOzZ4tlRDLv4qI= -k8s.io/kubectl v0.36.1/go.mod h1:/DGPAIewKsFWF9VFgGvkPhao2Ev4SNuE3BioZo8yPbk= -k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= -k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/kubectl v0.36.4 h1:xZd9g1bFBd7hpb1oKjK8lT9jRL18dtgr4DAPQG1Oksk= +k8s.io/kubectl v0.36.4/go.mod h1:STWlr78cdEa1hHpr55wpcboaqchvfDueKRNDa1zOd1w= +k8s.io/streaming v0.36.4 h1:RS5YlhrdBN2pKGVjgygGntdu6SNdsduyjGWGe3cX0vo= +k8s.io/streaming v0.36.4/go.mod h1:tJ6S2bZa2HxIBauguBbCWSCYyd93Grfz1+z3tcOvlDE= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -knative.dev/pkg v0.0.0-20260318013857-98d5a706d4fd h1:yeh+smYaouOwhkyCPj+AYACt1MeD+EI4mXSzSbmtj10= -knative.dev/pkg v0.0.0-20260318013857-98d5a706d4fd/go.mod h1:o/XS1E/hYh9IR8deEEiJG4kKtQfqnf9Gwt5bwp2x4AU= +knative.dev/pkg v0.0.0-20260622140654-39ebae2ee2dc h1:i2GrJuRdTFd4sNWocwefVGwuSOkOPjtadEWRTiGsEOY= +knative.dev/pkg v0.0.0-20260622140654-39ebae2ee2dc/go.mod h1:Ve19ZYW7DwIfQL4oCT9t9zmPp4egv0KacKVPXUcivDQ= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= @@ -356,7 +351,7 @@ sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7 sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/vendor/cel.dev/expr/README.md b/vendor/cel.dev/expr/README.md index 42d67f87c..f3bcbe4bf 100644 --- a/vendor/cel.dev/expr/README.md +++ b/vendor/cel.dev/expr/README.md @@ -47,7 +47,7 @@ highly recommended to serialize the type-checked expression as a protocol buffer. The CEL team will maintains canonical protocol buffers for ASTs and will keep these versions identical and wire-compatible in perpetuity: -* [CEL canonical](https://github.com/google/cel-spec/tree/master/proto/cel/expr) +* [CEL canonical](https://github.com/cel-expr/cel-spec/tree/master/proto/cel/expr) * [CEL v1alpha1](https://github.com/googleapis/googleapis/tree/master/google/api/expr/v1alpha1) diff --git a/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count b/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count deleted file mode 100644 index 2bcb9c7d1..000000000 Binary files a/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count and /dev/null differ diff --git a/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/weekends b/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/weekends deleted file mode 100644 index 0cfbf0888..000000000 --- a/vendor/github.com/cespare/xxhash/v2/~/Library/Application Support/go/telemetry/local/weekends +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE b/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go deleted file mode 100644 index a9e1b72ba..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ /dev/null @@ -1,756 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "os" - "path" - "runtime" - "strings" - "sync" - "sync/atomic" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/errgroup" -) - -type GzipHelperFunc func(io.Reader) (io.ReadCloser, error) - -type options struct { - chunkSize int - compressionLevel int - prioritizedFiles []string - missedPrioritizedFiles *[]string - compression Compression - ctx context.Context - minChunkSize int - gzipHelperFunc GzipHelperFunc -} - -type Option func(o *options) error - -// WithChunkSize option specifies the chunk size of eStargz blob to build. -func WithChunkSize(chunkSize int) Option { - return func(o *options) error { - o.chunkSize = chunkSize - return nil - } -} - -// WithCompressionLevel option specifies the gzip compression level. -// The default is gzip.BestCompression. -// This option will be ignored if WithCompression option is used. -// See also: https://godoc.org/compress/gzip#pkg-constants -func WithCompressionLevel(level int) Option { - return func(o *options) error { - o.compressionLevel = level - return nil - } -} - -// WithPrioritizedFiles option specifies the list of prioritized files. -// These files must be complete paths that are absolute or relative to "/" -// For example, all of "foo/bar", "/foo/bar", "./foo/bar" and "../foo/bar" -// are treated as "/foo/bar". -func WithPrioritizedFiles(files []string) Option { - return func(o *options) error { - o.prioritizedFiles = files - return nil - } -} - -// WithAllowPrioritizeNotFound makes Build continue the execution even if some -// of prioritized files specified by WithPrioritizedFiles option aren't found -// in the input tar. Instead, this records all missed file names to the passed -// slice. -func WithAllowPrioritizeNotFound(missedFiles *[]string) Option { - return func(o *options) error { - if missedFiles == nil { - return fmt.Errorf("WithAllowPrioritizeNotFound: slice must be passed") - } - o.missedPrioritizedFiles = missedFiles - return nil - } -} - -// WithCompression specifies compression algorithm to be used. -// Default is gzip. -func WithCompression(compression Compression) Option { - return func(o *options) error { - o.compression = compression - return nil - } -} - -// WithContext specifies a context that can be used for clean canceleration. -func WithContext(ctx context.Context) Option { - return func(o *options) error { - o.ctx = ctx - return nil - } -} - -// WithMinChunkSize option specifies the minimal number of bytes of data -// must be written in one gzip stream. -// By increasing this number, one gzip stream can contain multiple files -// and it hopefully leads to smaller result blob. -// NOTE: This adds a TOC property that old reader doesn't understand. -func WithMinChunkSize(minChunkSize int) Option { - return func(o *options) error { - o.minChunkSize = minChunkSize - return nil - } -} - -// WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers. -// When a gzip-compressed layer is detected, this function will be used instead of the -// Go standard library gzip decompression for better performance. -// The function should take an io.Reader as input and return an io.ReadCloser. -// If nil, the Go standard library gzip.NewReader will be used. -func WithGzipHelperFunc(gzipHelperFunc GzipHelperFunc) Option { - return func(o *options) error { - o.gzipHelperFunc = gzipHelperFunc - return nil - } -} - -// Blob is an eStargz blob. -type Blob struct { - io.ReadCloser - diffID digest.Digester - tocDigest digest.Digest - readCompleted *atomic.Bool - uncompressedSize *atomic.Int64 -} - -// DiffID returns the digest of uncompressed blob. -// It is only valid to call DiffID after Close. -func (b *Blob) DiffID() digest.Digest { - return b.diffID.Digest() -} - -// TOCDigest returns the digest of uncompressed TOC JSON. -func (b *Blob) TOCDigest() digest.Digest { - return b.tocDigest -} - -// UncompressedSize returns the size of uncompressed blob. -// UncompressedSize should only be called after the blob has been fully read. -func (b *Blob) UncompressedSize() (int64, error) { - switch { - case b.uncompressedSize == nil || b.readCompleted == nil: - return -1, fmt.Errorf("readCompleted or uncompressedSize is not initialized") - case !b.readCompleted.Load(): - return -1, fmt.Errorf("called UncompressedSize before the blob has been fully read") - default: - return b.uncompressedSize.Load(), nil - } -} - -// Build builds an eStargz blob which is an extended version of stargz, from a blob (gzip, zstd -// or plain tar) passed through the argument. If there are some prioritized files are listed in -// the option, these files are grouped as "prioritized" and can be used for runtime optimization -// (e.g. prefetch). This function builds a blob in parallel, with dividing that blob into several -// (at least the number of runtime.GOMAXPROCS(0)) sub-blobs. -func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { - var opts options - opts.compressionLevel = gzip.BestCompression // BestCompression by default - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - if opts.compression == nil { - opts.compression = newGzipCompressionWithLevel(opts.compressionLevel) - } - layerFiles := newTempFiles() - ctx := opts.ctx - if ctx == nil { - ctx = context.Background() - } - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-done: - // nop - case <-ctx.Done(): - layerFiles.CleanupAll() - } - }() - defer func() { - if rErr != nil { - if err := layerFiles.CleanupAll(); err != nil { - rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr) - } - } - if cErr := ctx.Err(); cErr != nil { - rErr = fmt.Errorf("error from context %q: %w", cErr, rErr) - } - }() - tarBlob, err := decompressBlob(tarBlob, layerFiles, opts.gzipHelperFunc) - if err != nil { - return nil, err - } - entries, err := sortEntries(tarBlob, opts.prioritizedFiles, opts.missedPrioritizedFiles) - if err != nil { - return nil, err - } - var tarParts [][]*entry - if opts.minChunkSize > 0 { - // Each entry needs to know the size of the current gzip stream so they - // cannot be processed in parallel. - tarParts = [][]*entry{entries} - } else { - tarParts = divideEntries(entries, runtime.GOMAXPROCS(0)) - } - writers := make([]*Writer, len(tarParts)) - payloads := make([]*os.File, len(tarParts)) - var mu sync.Mutex - var eg errgroup.Group - for i, parts := range tarParts { - i, parts := i, parts - // builds verifiable stargz sub-blobs - eg.Go(func() error { - esgzFile, err := layerFiles.TempFile("", "esgzdata") - if err != nil { - return err - } - sw := NewWriterWithCompressor(esgzFile, opts.compression) - sw.ChunkSize = opts.chunkSize - sw.MinChunkSize = opts.minChunkSize - if sw.needsOpenGzEntries == nil { - sw.needsOpenGzEntries = make(map[string]struct{}) - } - for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} { - sw.needsOpenGzEntries[f] = struct{}{} - } - if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { - return err - } - mu.Lock() - writers[i] = sw - payloads[i] = esgzFile - mu.Unlock() - return nil - }) - } - if err := eg.Wait(); err != nil { - rErr = err - return nil, err - } - tocAndFooter, tocDgst, err := closeWithCombine(writers...) - if err != nil { - rErr = err - return nil, err - } - var rs []io.Reader - for _, p := range payloads { - fs, err := fileSectionReader(p) - if err != nil { - return nil, err - } - rs = append(rs, fs) - } - diffID := digest.Canonical.Digester() - pr, pw := io.Pipe() - readCompleted := new(atomic.Bool) - uncompressedSize := new(atomic.Int64) - go func() { - var size int64 - var decompressFunc func(io.Reader) (io.ReadCloser, error) - if _, ok := opts.compression.(*gzipCompression); ok && opts.gzipHelperFunc != nil { - decompressFunc = opts.gzipHelperFunc - } else { - decompressFunc = opts.compression.Reader - } - decompressR, err := decompressFunc(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) - if err != nil { - pw.CloseWithError(err) - return - } - defer decompressR.Close() - if size, err = io.Copy(diffID.Hash(), decompressR); err != nil { - pw.CloseWithError(err) - return - } - uncompressedSize.Store(size) - readCompleted.Store(true) - pw.Close() - }() - return &Blob{ - ReadCloser: readCloser{ - Reader: pr, - closeFunc: layerFiles.CleanupAll, - }, - tocDigest: tocDgst, - diffID: diffID, - readCompleted: readCompleted, - uncompressedSize: uncompressedSize, - }, nil -} - -// closeWithCombine takes unclosed Writers and close them. This also returns the -// toc that combined all Writers into. -// Writers doesn't write TOC and footer to the underlying writers so they can be -// combined into a single eStargz and tocAndFooter returned by this function can -// be appended at the tail of that combined blob. -func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) { - if len(ws) == 0 { - return nil, "", fmt.Errorf("at least one writer must be passed") - } - for _, w := range ws { - if w.closed { - return nil, "", fmt.Errorf("writer must be unclosed") - } - defer func(w *Writer) { w.closed = true }(w) - if err := w.closeGz(); err != nil { - return nil, "", err - } - if err := w.bw.Flush(); err != nil { - return nil, "", err - } - } - var ( - mtoc = new(JTOC) - currentOffset int64 - ) - mtoc.Version = ws[0].toc.Version - for _, w := range ws { - for _, e := range w.toc.Entries { - // Recalculate Offset of non-empty files/chunks - if (e.Type == "reg" && e.Size > 0) || e.Type == "chunk" { - e.Offset += currentOffset - } - mtoc.Entries = append(mtoc.Entries, e) - } - if w.toc.Version > mtoc.Version { - mtoc.Version = w.toc.Version - } - currentOffset += w.cw.n - } - - return tocAndFooter(ws[0].compressor, mtoc, currentOffset) -} - -func tocAndFooter(compressor Compressor, toc *JTOC, offset int64) (io.Reader, digest.Digest, error) { - buf := new(bytes.Buffer) - tocDigest, err := compressor.WriteTOCAndFooter(buf, offset, toc, nil) - if err != nil { - return nil, "", err - } - return buf, tocDigest, nil -} - -// divideEntries divides passed entries to the parts at least the number specified by the -// argument. -func divideEntries(entries []*entry, minPartsNum int) (set [][]*entry) { - var estimatedSize int64 - for _, e := range entries { - estimatedSize += e.header.Size - } - unitSize := estimatedSize / int64(minPartsNum) - var ( - nextEnd = unitSize - offset int64 - ) - set = append(set, []*entry{}) - for _, e := range entries { - set[len(set)-1] = append(set[len(set)-1], e) - offset += e.header.Size - if offset > nextEnd { - set = append(set, []*entry{}) - nextEnd += unitSize - } - } - return -} - -var errNotFound = errors.New("not found") - -// sortEntries reads the specified tar blob and returns a list of tar entries. -// If some of prioritized files are specified, the list starts from these -// files with keeping the order specified by the argument. -func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]string) ([]*entry, error) { - - // Import tar file. - intar, err := importTar(in) - if err != nil { - return nil, fmt.Errorf("failed to sort: %w", err) - } - - // Sort the tar file respecting to the prioritized files list. - sorted := &tarFile{} - picked := make(map[string]struct{}) - for _, l := range prioritized { - if err := moveRec(l, intar, sorted, picked); err != nil { - if errors.Is(err, errNotFound) && missedPrioritized != nil { - *missedPrioritized = append(*missedPrioritized, l) - continue // allow not found - } - return nil, fmt.Errorf("failed to sort tar entries: %w", err) - } - } - if len(prioritized) == 0 { - sorted.add(&entry{ - header: &tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } else { - sorted.add(&entry{ - header: &tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } - - // Dump prioritized entries followed by the rest entries while skipping picked ones. - return append(sorted.dump(nil), intar.dump(picked)...), nil -} - -// readerFromEntries returns a reader of tar archive that contains entries passed -// through the arguments. -func readerFromEntries(entries ...*entry) io.Reader { - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - defer tw.Close() - for _, entry := range entries { - if err := tw.WriteHeader(entry.header); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar header: %v", err)) - return - } - if _, err := io.Copy(tw, entry.payload); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar payload: %v", err)) - return - } - } - pw.Close() - }() - return pr -} - -func importTar(in io.ReaderAt) (*tarFile, error) { - tf := &tarFile{} - pw, err := newCountReadSeeker(in) - if err != nil { - return nil, fmt.Errorf("failed to make position watcher: %w", err) - } - tr := tar.NewReader(pw) - - // Walk through all nodes. - for { - // Fetch and parse next header. - h, err := tr.Next() - if err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("failed to parse tar file, %w", err) - } - switch cleanEntryName(h.Name) { - case PrefetchLandmark, NoPrefetchLandmark: - // Ignore existing landmark - continue - } - - // Add entry. If it already exists, replace it. - if _, ok := tf.get(h.Name); ok { - tf.remove(h.Name) - } - tf.add(&entry{ - header: h, - payload: io.NewSectionReader(in, pw.currentPos(), h.Size), - }) - } - - return tf, nil -} - -func moveRec(name string, in *tarFile, out *tarFile, picked map[string]struct{}) error { - name = cleanEntryName(name) - if name == "" { // root directory. stop recursion. - if e, ok := in.get(name); ok { - // entry of the root directory exists. we should move it as well. - // this case will occur if tar entries are prefixed with "./", "/", etc. - if _, done := picked[name]; !done { - out.add(e) - picked[name] = struct{}{} - } - } - return nil - } - - _, okIn := in.get(name) - _, okOut := out.get(name) - _, okPicked := picked[name] - if !okIn && !okOut && !okPicked { - return fmt.Errorf("file: %q: %w", name, errNotFound) - } - - parent, _ := path.Split(strings.TrimSuffix(name, "/")) - if err := moveRec(parent, in, out, picked); err != nil { - return err - } - if e, ok := in.get(name); ok && e.header.Typeflag == tar.TypeLink { - if err := moveRec(e.header.Linkname, in, out, picked); err != nil { - return err - } - } - if _, done := picked[name]; done { - return nil - } - if e, ok := in.get(name); ok { - out.add(e) - picked[name] = struct{}{} - } - return nil -} - -type entry struct { - header *tar.Header - payload io.ReadSeeker -} - -type tarFile struct { - index map[string]*entry - stream []*entry -} - -func (f *tarFile) add(e *entry) { - if f.index == nil { - f.index = make(map[string]*entry) - } - f.index[cleanEntryName(e.header.Name)] = e - f.stream = append(f.stream, e) -} - -func (f *tarFile) remove(name string) { - name = cleanEntryName(name) - if f.index != nil { - delete(f.index, name) - } - var filtered []*entry - for _, e := range f.stream { - if cleanEntryName(e.header.Name) == name { - continue - } - filtered = append(filtered, e) - } - f.stream = filtered -} - -func (f *tarFile) get(name string) (e *entry, ok bool) { - if f.index == nil { - return nil, false - } - e, ok = f.index[cleanEntryName(name)] - return -} - -func (f *tarFile) dump(skip map[string]struct{}) []*entry { - if len(skip) == 0 { - return f.stream - } - var out []*entry - for _, e := range f.stream { - if _, ok := skip[cleanEntryName(e.header.Name)]; ok { - continue - } - out = append(out, e) - } - return out -} - -type readCloser struct { - io.Reader - closeFunc func() error -} - -func (rc readCloser) Close() error { - return rc.closeFunc() -} - -func fileSectionReader(file *os.File) (*io.SectionReader, error) { - info, err := file.Stat() - if err != nil { - return nil, err - } - return io.NewSectionReader(file, 0, info.Size()), nil -} - -func newTempFiles() *tempFiles { - return &tempFiles{} -} - -type tempFiles struct { - files []*os.File - filesMu sync.Mutex - cleanupOnce sync.Once -} - -func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) { - f, err := os.CreateTemp(dir, pattern) - if err != nil { - return nil, err - } - tf.filesMu.Lock() - tf.files = append(tf.files, f) - tf.filesMu.Unlock() - return f, nil -} - -func (tf *tempFiles) CleanupAll() (err error) { - tf.cleanupOnce.Do(func() { - err = tf.cleanupAll() - }) - return -} - -func (tf *tempFiles) cleanupAll() error { - tf.filesMu.Lock() - defer tf.filesMu.Unlock() - var allErr []error - for _, f := range tf.files { - if err := f.Close(); err != nil { - allErr = append(allErr, err) - } - if err := os.Remove(f.Name()); err != nil { - allErr = append(allErr, err) - } - } - tf.files = nil - return errorutil.Aggregate(allErr) -} - -func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) { - pos := int64(0) - return &countReadSeeker{r: r, cPos: &pos}, nil -} - -type countReadSeeker struct { - r io.ReaderAt - cPos *int64 - - mu sync.Mutex -} - -func (cr *countReadSeeker) Read(p []byte) (int, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - n, err := cr.r.ReadAt(p, *cr.cPos) - if err == nil { - *cr.cPos += int64(n) - } - return n, err -} - -func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - switch whence { - default: - return 0, fmt.Errorf("unknown whence: %v", whence) - case io.SeekStart: - case io.SeekCurrent: - offset += *cr.cPos - case io.SeekEnd: - return 0, fmt.Errorf("unsupported whence: %v", whence) - } - - if offset < 0 { - return 0, fmt.Errorf("invalid offset") - } - *cr.cPos = offset - return offset, nil -} - -func (cr *countReadSeeker) currentPos() int64 { - cr.mu.Lock() - defer cr.mu.Unlock() - - return *cr.cPos -} - -func decompressBlob(org *io.SectionReader, tmp *tempFiles, gzipHelperFunc GzipHelperFunc) (*io.SectionReader, error) { - if org.Size() < 4 { - return org, nil - } - src := make([]byte, 4) - if _, err := org.Read(src); err != nil && err != io.EOF { - return nil, err - } - var dR io.Reader - if bytes.Equal([]byte{0x1F, 0x8B, 0x08}, src[:3]) { - // gzip - var dgR io.ReadCloser - var err error - if gzipHelperFunc != nil { - dgR, err = gzipHelperFunc(io.NewSectionReader(org, 0, org.Size())) - } else { - dgR, err = gzip.NewReader(io.NewSectionReader(org, 0, org.Size())) - } - if err != nil { - return nil, err - } - defer dgR.Close() - dR = io.Reader(dgR) - } else if bytes.Equal([]byte{0x28, 0xb5, 0x2f, 0xfd}, src[:4]) { - // zstd - dzR, err := zstd.NewReader(io.NewSectionReader(org, 0, org.Size())) - if err != nil { - return nil, err - } - defer dzR.Close() - dR = io.Reader(dzR) - } else { - // uncompressed - return io.NewSectionReader(org, 0, org.Size()), nil - } - b, err := tmp.TempFile("", "uncompresseddata") - if err != nil { - return nil, err - } - if _, err := io.Copy(b, dR); err != nil { - return nil, err - } - return fileSectionReader(b) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go deleted file mode 100644 index 6de78b02d..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errorutil - -import ( - "errors" - "fmt" - "strings" -) - -// Aggregate combines a list of errors into a single new error. -func Aggregate(errs []error) error { - switch len(errs) { - case 0: - return nil - case 1: - return errs[0] - default: - points := make([]string, len(errs)+1) - points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) - for i, err := range errs { - points[i+1] = fmt.Sprintf("* %s", err) - } - return errors.New(strings.Join(points, "\n\t")) - } -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go deleted file mode 100644 index 693730420..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ /dev/null @@ -1,1232 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "bufio" - "bytes" - "compress/gzip" - "crypto/sha256" - "errors" - "fmt" - "hash" - "io" - "os" - "path" - "sort" - "strings" - "sync" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - digest "github.com/opencontainers/go-digest" - "github.com/vbatts/tar-split/archive/tar" -) - -// A Reader permits random access reads from a stargz file. -type Reader struct { - sr *io.SectionReader - toc *JTOC - tocDigest digest.Digest - - // m stores all non-chunk entries, keyed by name. - m map[string]*TOCEntry - - // chunks stores all TOCEntry values for regular files that - // are split up. For a file with a single chunk, it's only - // stored in m. - chunks map[string][]*TOCEntry - - decompressor Decompressor -} - -type openOpts struct { - tocOffset int64 - decompressors []Decompressor - telemetry *Telemetry -} - -// OpenOption is an option used during opening the layer -type OpenOption func(o *openOpts) error - -// WithTOCOffset option specifies the offset of TOC -func WithTOCOffset(tocOffset int64) OpenOption { - return func(o *openOpts) error { - o.tocOffset = tocOffset - return nil - } -} - -// WithDecompressors option specifies decompressors to use. -// Default is gzip-based decompressor. -func WithDecompressors(decompressors ...Decompressor) OpenOption { - return func(o *openOpts) error { - o.decompressors = decompressors - return nil - } -} - -// WithTelemetry option specifies the telemetry hooks -func WithTelemetry(telemetry *Telemetry) OpenOption { - return func(o *openOpts) error { - o.telemetry = telemetry - return nil - } -} - -// MeasureLatencyHook is a func which takes start time and records the diff -type MeasureLatencyHook func(time.Time) - -// Telemetry is a struct which defines telemetry hooks. By implementing these hooks you should be able to record -// the latency metrics of the respective steps of estargz open operation. To be used with estargz.OpenWithTelemetry(...) -type Telemetry struct { - GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds) - GetTocLatency MeasureLatencyHook // measure time to GET TOC JSON (in milliseconds) - DeserializeTocLatency MeasureLatencyHook // measure time to deserialize TOC JSON (in milliseconds) -} - -// Open opens a stargz file for reading. -// The behavior is configurable using options. -// -// Note that each entry name is normalized as the path that is relative to root. -func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { - var opts openOpts - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - - gzipCompressors := []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} - decompressors := append(gzipCompressors, opts.decompressors...) - - // Determine the size to fetch. Try to fetch as many bytes as possible. - fetchSize := maxFooterSize(sr.Size(), decompressors...) - if maybeTocOffset := opts.tocOffset; maybeTocOffset > fetchSize { - if maybeTocOffset > sr.Size() { - return nil, fmt.Errorf("blob size %d is smaller than the toc offset", sr.Size()) - } - fetchSize = sr.Size() - maybeTocOffset - } - - start := time.Now() // before getting layer footer - footer := make([]byte, fetchSize) - if _, err := sr.ReadAt(footer, sr.Size()-fetchSize); err != nil { - return nil, fmt.Errorf("error reading footer: %v", err) - } - if opts.telemetry != nil && opts.telemetry.GetFooterLatency != nil { - opts.telemetry.GetFooterLatency(start) - } - - var allErr []error - var found bool - var r *Reader - for _, d := range decompressors { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - maybeTocBytes := footer[:fOffset] - _, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) - if err != nil { - allErr = append(allErr, err) - continue - } - if tocOffset >= 0 && tocSize <= 0 { - tocSize = sr.Size() - tocOffset - fSize - } - if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) { - maybeTocBytes = maybeTocBytes[:tocSize] - } - r, err = parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes, opts) - if err == nil { - found = true - break - } - allErr = append(allErr, err) - } - if !found { - return nil, errorutil.Aggregate(allErr) - } - if err := r.initFields(); err != nil { - return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) - } - return r, nil -} - -// OpenFooter extracts and parses footer from the given blob. -// only supports gzip-based eStargz. -func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr error) { - if sr.Size() < FooterSize && sr.Size() < legacyFooterSize { - return 0, 0, fmt.Errorf("blob size %d is smaller than the footer size", sr.Size()) - } - var footer [FooterSize]byte - if _, err := sr.ReadAt(footer[:], sr.Size()-FooterSize); err != nil { - return 0, 0, fmt.Errorf("error reading footer: %v", err) - } - var allErr []error - for _, d := range []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - _, tocOffset, _, err := d.ParseFooter(footer[fOffset:]) - if err == nil { - return tocOffset, fSize, err - } - allErr = append(allErr, err) - } - return 0, 0, errorutil.Aggregate(allErr) -} - -// initFields populates the Reader from r.toc after decoding it from -// JSON. -// -// Unexported fields are populated and TOCEntry fields that were -// implicit in the JSON are populated. -func (r *Reader) initFields() error { - r.m = make(map[string]*TOCEntry, len(r.toc.Entries)) - r.chunks = make(map[string][]*TOCEntry) - var lastPath string - uname := map[int]string{} - gname := map[int]string{} - var lastRegEnt *TOCEntry - var chunkTopIndex int - for i, ent := range r.toc.Entries { - ent.Name = cleanEntryName(ent.Name) - switch ent.Type { - case "reg", "chunk": - if ent.Offset != r.toc.Entries[chunkTopIndex].Offset { - chunkTopIndex = i - } - ent.chunkTopIndex = chunkTopIndex - } - if ent.Type == "reg" { - lastRegEnt = ent - } - if ent.Type == "chunk" { - ent.Name = lastPath - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - if ent.ChunkSize == 0 && lastRegEnt != nil { - ent.ChunkSize = lastRegEnt.Size - ent.ChunkOffset - } - } else { - lastPath = ent.Name - - if ent.Uname != "" { - uname[ent.UID] = ent.Uname - } else { - ent.Uname = uname[ent.UID] - } - if ent.Gname != "" { - gname[ent.GID] = ent.Gname - } else { - ent.Gname = gname[ent.GID] - } - - ent.modTime, _ = time.Parse(time.RFC3339, ent.ModTime3339) - - if ent.Type == "dir" { - ent.NumLink++ // Parent dir links to this directory - } - r.m[ent.Name] = ent - } - if ent.Type == "reg" && ent.ChunkSize > 0 && ent.ChunkSize < ent.Size { - r.chunks[ent.Name] = make([]*TOCEntry, 0, ent.Size/ent.ChunkSize+1) - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - } - if ent.ChunkSize == 0 && ent.Size != 0 { - ent.ChunkSize = ent.Size - } - } - - // Populate children, add implicit directories: - for _, ent := range r.toc.Entries { - if ent.Type == "chunk" { - continue - } - // add "foo/": - // add "foo" child to "" (creating "" if necessary) - // - // add "foo/bar/": - // add "bar" child to "foo" (creating "foo" if necessary) - // - // add "foo/bar.txt": - // add "bar.txt" child to "foo" (creating "foo" if necessary) - // - // add "a/b/c/d/e/f.txt": - // create "a/b/c/d/e" node - // add "f.txt" child to "e" - - name := ent.Name - pdirName := parentDir(name) - if name == pdirName { - // This entry and its parent are the same. - // Ignore this for avoiding infinite loop of the reference. - // The example case where this can occur is when tar contains the root - // directory itself (e.g. "./", "/"). - continue - } - pdir := r.getOrCreateDir(pdirName) - ent.NumLink++ // at least one name(ent.Name) references this entry. - if ent.Type == "hardlink" { - org, err := r.getSource(ent) - if err != nil { - return err - } - org.NumLink++ // original entry is referenced by this ent.Name. - ent = org - } - pdir.addChild(path.Base(name), ent) - } - - lastOffset := r.sr.Size() - for i := len(r.toc.Entries) - 1; i >= 0; i-- { - e := r.toc.Entries[i] - if e.isDataType() { - e.nextOffset = lastOffset - } - if e.Offset != 0 && e.InnerOffset == 0 { - lastOffset = e.Offset - } - } - - if len(r.m) == 0 { - r.m[""] = &TOCEntry{ - Name: "", - Type: "dir", - Mode: 0755, - NumLink: 1, - } - } - - return nil -} - -func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) { - if ent.Type == "hardlink" { - org, ok := r.m[cleanEntryName(ent.LinkName)] - if !ok { - return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName) - } - ent, err = r.getSource(org) - if err != nil { - return nil, err - } - } - return ent, nil -} - -func parentDir(p string) string { - dir, _ := path.Split(p) - return strings.TrimSuffix(dir, "/") -} - -func (r *Reader) getOrCreateDir(d string) *TOCEntry { - e, ok := r.m[d] - if !ok { - e = &TOCEntry{ - Name: d, - Type: "dir", - Mode: 0755, - NumLink: 2, // The directory itself(.) and the parent link to this directory. - } - r.m[d] = e - if d != "" { - pdir := r.getOrCreateDir(parentDir(d)) - pdir.addChild(path.Base(d), e) - } - } - return e -} - -func (r *Reader) TOCDigest() digest.Digest { - return r.tocDigest -} - -// VerifyTOC checks that the TOC JSON in the passed blob matches the -// passed digests and that the TOC JSON contains digests for all chunks -// contained in the blob. If the verification succceeds, this function -// returns TOCEntryVerifier which holds all chunk digests in the stargz blob. -func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) { - // Verify the digest of TOC JSON - if r.tocDigest != tocDigest { - return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest) - } - return r.Verifiers() -} - -// Verifiers returns TOCEntryVerifier of this chunk. Use VerifyTOC instead in most cases -// because this doesn't verify TOC. -func (r *Reader) Verifiers() (TOCEntryVerifier, error) { - chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest - regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest - var chunkDigestMapIncomplete bool - var regDigestMapIncomplete bool - var containsChunk bool - for _, e := range r.toc.Entries { - if e.Type != "reg" && e.Type != "chunk" { - continue - } - - // offset must be unique in stargz blob - _, dOK := chunkDigestMap[e.Offset] - _, rOK := regDigestMap[e.Offset] - if dOK || rOK { - return nil, fmt.Errorf("offset %d found twice", e.Offset) - } - - if e.Type == "reg" { - if e.Size == 0 { - continue // ignores empty file - } - - // record the digest of regular file payload - if e.Digest != "" { - d, err := digest.Parse(e.Digest) - if err != nil { - return nil, fmt.Errorf("failed to parse regular file digest %q: %w", e.Digest, err) - } - regDigestMap[e.Offset] = d - } else { - regDigestMapIncomplete = true - } - } else { - containsChunk = true // this layer contains "chunk" entries. - } - - // "reg" also can contain ChunkDigest (e.g. when "reg" is the first entry of - // chunked file) - if e.ChunkDigest != "" { - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, fmt.Errorf("failed to parse chunk digest %q: %w", e.ChunkDigest, err) - } - chunkDigestMap[e.Offset] = d - } else { - chunkDigestMapIncomplete = true - } - } - - if chunkDigestMapIncomplete { - // Though some chunk digests are not found, if this layer doesn't contain - // "chunk"s and all digest of "reg" files are recorded, we can use them instead. - if !containsChunk && !regDigestMapIncomplete { - return &verifier{digestMap: regDigestMap}, nil - } - return nil, fmt.Errorf("some ChunkDigest not found in TOC JSON") - } - - return &verifier{digestMap: chunkDigestMap}, nil -} - -// verifier is an implementation of TOCEntryVerifier which holds verifiers keyed by -// offset of the chunk. -type verifier struct { - digestMap map[int64]digest.Digest - digestMapMu sync.Mutex -} - -// Verifier returns a content verifier specified by TOCEntry. -func (v *verifier) Verifier(ce *TOCEntry) (digest.Verifier, error) { - v.digestMapMu.Lock() - defer v.digestMapMu.Unlock() - d, ok := v.digestMap[ce.Offset] - if !ok { - return nil, fmt.Errorf("verifier for offset=%d,size=%d hasn't been registered", - ce.Offset, ce.ChunkSize) - } - return d.Verifier(), nil -} - -// ChunkEntryForOffset returns the TOCEntry containing the byte of the -// named file at the given offset within the file. -// Name must be absolute path or one that is relative to root. -func (r *Reader) ChunkEntryForOffset(name string, offset int64) (e *TOCEntry, ok bool) { - name = cleanEntryName(name) - e, ok = r.Lookup(name) - if !ok || !e.isDataType() { - return nil, false - } - ents := r.chunks[name] - if len(ents) < 2 { - if offset >= e.ChunkSize { - return nil, false - } - return e, true - } - i := sort.Search(len(ents), func(i int) bool { - e := ents[i] - return e.ChunkOffset >= offset || (offset > e.ChunkOffset && offset < e.ChunkOffset+e.ChunkSize) - }) - if i == len(ents) { - return nil, false - } - return ents[i], true -} - -// Lookup returns the Table of Contents entry for the given path. -// -// To get the root directory, use the empty string. -// Path must be absolute path or one that is relative to root. -func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) { - path = cleanEntryName(path) - if r == nil { - return - } - e, ok = r.m[path] - if ok && e.Type == "hardlink" { - var err error - e, err = r.getSource(e) - if err != nil { - return nil, false - } - } - return -} - -// OpenFile returns the reader of the specified file payload. -// -// Name must be absolute path or one that is relative to root. -func (r *Reader) OpenFile(name string) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) newFileReader(name string) (*fileReader, error) { - name = cleanEntryName(name) - ent, ok := r.Lookup(name) - if !ok { - // TODO: come up with some error plan. This is lazy: - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: os.ErrNotExist, - } - } - if ent.Type != "reg" { - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: errors.New("not a regular file"), - } - } - return &fileReader{ - r: r, - size: ent.Size, - ents: r.getChunks(ent), - }, nil -} - -func (r *Reader) OpenFileWithPreReader(name string, preRead func(*TOCEntry, io.Reader) error) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - fr.preRead = preRead - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) getChunks(ent *TOCEntry) []*TOCEntry { - if ents, ok := r.chunks[ent.Name]; ok { - return ents - } - return []*TOCEntry{ent} -} - -type fileReader struct { - r *Reader - size int64 - ents []*TOCEntry // 1 or more reg/chunk entries - preRead func(*TOCEntry, io.Reader) error -} - -func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { - if off >= fr.size { - return 0, io.EOF - } - if off < 0 { - return 0, errors.New("invalid offset") - } - var i int - if len(fr.ents) > 1 { - i = sort.Search(len(fr.ents), func(i int) bool { - return fr.ents[i].ChunkOffset >= off - }) - if i == len(fr.ents) { - i = len(fr.ents) - 1 - } - } - ent := fr.ents[i] - if ent.ChunkOffset > off { - if i == 0 { - return 0, errors.New("internal error; first chunk offset is non-zero") - } - ent = fr.ents[i-1] - } - - // If ent is a chunk of a large file, adjust the ReadAt - // offset by the chunk's offset. - off -= ent.ChunkOffset - - finalEnt := fr.ents[len(fr.ents)-1] - compressedOff := ent.Offset - // compressedBytesRemain is the number of compressed bytes in this - // file remaining, over 1+ chunks. - compressedBytesRemain := finalEnt.NextOffset() - compressedOff - - sr := io.NewSectionReader(fr.r.sr, compressedOff, compressedBytesRemain) - - const maxRead = 2 << 20 - var bufSize = maxRead - if compressedBytesRemain < maxRead { - bufSize = int(compressedBytesRemain) - } - - br := bufio.NewReaderSize(sr, bufSize) - if _, err := br.Peek(bufSize); err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.peek: %v", err) - } - - dr, err := fr.r.decompressor.Reader(br) - if err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err) - } - defer dr.Close() - - if fr.preRead == nil { - if n, err := io.CopyN(io.Discard, dr, ent.InnerOffset+off); n != ent.InnerOffset+off || err != nil { - return 0, fmt.Errorf("discard of %d bytes != %v, %v", ent.InnerOffset+off, n, err) - } - return io.ReadFull(dr, p) - } - - var retN int - var retErr error - var found bool - var nr int64 - for _, e := range fr.r.toc.Entries[ent.chunkTopIndex:] { - if !e.isDataType() { - continue - } - if e.Offset != fr.r.toc.Entries[ent.chunkTopIndex].Offset { - break - } - if in, err := io.CopyN(io.Discard, dr, e.InnerOffset-nr); err != nil || in != e.InnerOffset-nr { - return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.InnerOffset-nr, in, err) - } - nr = e.InnerOffset - if e == ent { - found = true - if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil { - return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err) - } - retN, retErr = io.ReadFull(dr, p) - nr += off + int64(retN) - continue - } - cr := &countReader{r: io.LimitReader(dr, e.ChunkSize)} - if err := fr.preRead(e, cr); err != nil { - return 0, fmt.Errorf("failed to pre read: %w", err) - } - nr += cr.n - } - if !found { - return 0, fmt.Errorf("fileReader.ReadAt: target entry not found") - } - return retN, retErr -} - -// A Writer writes stargz files. -// -// Use NewWriter to create a new Writer. -type Writer struct { - bw *bufio.Writer - cw *countWriter - toc *JTOC - diffHash hash.Hash // SHA-256 of uncompressed tar - - closed bool - gz io.WriteCloser - lastUsername map[int]string - lastGroupname map[int]string - compressor Compressor - - uncompressedCounter *countWriteFlusher - - // ChunkSize optionally controls the maximum number of bytes - // of data of a regular file that can be written in one gzip - // stream before a new gzip stream is started. - // Zero means to use a default, currently 4 MiB. - ChunkSize int - - // MinChunkSize optionally controls the minimum number of bytes - // of data must be written in one gzip stream before a new gzip - // NOTE: This adds a TOC property that stargz snapshotter < v0.13.0 doesn't understand. - MinChunkSize int - - needsOpenGzEntries map[string]struct{} -} - -// currentCompressionWriter writes to the current w.gz field, which can -// change throughout writing a tar entry. -// -// Additionally, it updates w's SHA-256 of the uncompressed bytes -// of the tar file. -type currentCompressionWriter struct{ w *Writer } - -func (ccw currentCompressionWriter) Write(p []byte) (int, error) { - ccw.w.diffHash.Write(p) - if ccw.w.gz == nil { - if err := ccw.w.condOpenGz(); err != nil { - return 0, err - } - } - return ccw.w.gz.Write(p) -} - -func (w *Writer) chunkSize() int { - if w.ChunkSize <= 0 { - return 4 << 20 - } - return w.ChunkSize -} - -// Unpack decompresses the given estargz blob and returns a ReadCloser of the tar blob. -// TOC JSON and footer are removed. -func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) { - footerSize := c.FooterSize() - if sr.Size() < footerSize { - return nil, fmt.Errorf("blob is too small; %d < %d", sr.Size(), footerSize) - } - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - return nil, err - } - blobPayloadSize, _, _, err := c.ParseFooter(footer) - if err != nil { - return nil, fmt.Errorf("failed to parse footer: %w", err) - } - if blobPayloadSize < 0 { - blobPayloadSize = sr.Size() - } - return c.Reader(io.LimitReader(sr, blobPayloadSize)) -} - -// NewWriter returns a new stargz writer (gzip-based) writing to w. -// -// The writer must be closed to write its trailing table of contents. -func NewWriter(w io.Writer) *Writer { - return NewWriterLevel(w, gzip.BestCompression) -} - -// NewWriterLevel returns a new stargz writer (gzip-based) writing to w. -// The compression level is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { - return NewWriterWithCompressor(w, NewGzipCompressorWithLevel(compressionLevel)) -} - -// NewWriterWithCompressor returns a new stargz writer writing to w. -// The compression method is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer { - bw := bufio.NewWriter(w) - cw := &countWriter{w: bw} - return &Writer{ - bw: bw, - cw: cw, - toc: &JTOC{Version: 1}, - diffHash: sha256.New(), - compressor: c, - uncompressedCounter: &countWriteFlusher{}, - } -} - -// Close writes the stargz's table of contents and flushes all the -// buffers, returning any error. -func (w *Writer) Close() (digest.Digest, error) { - if w.closed { - return "", nil - } - defer func() { w.closed = true }() - - if err := w.closeGz(); err != nil { - return "", err - } - - // Write the TOC index and footer. - tocDigest, err := w.compressor.WriteTOCAndFooter(w.cw, w.cw.n, w.toc, w.diffHash) - if err != nil { - return "", err - } - if err := w.bw.Flush(); err != nil { - return "", err - } - - return tocDigest, nil -} - -func (w *Writer) closeGz() error { - if w.closed { - return errors.New("write on closed Writer") - } - if w.gz != nil { - if err := w.gz.Close(); err != nil { - return err - } - w.gz = nil - } - return nil -} - -func (w *Writer) flushGz() error { - if w.closed { - return errors.New("flush on closed Writer") - } - if w.gz != nil { - if f, ok := w.gz.(interface { - Flush() error - }); ok { - return f.Flush() - } - } - return nil -} - -// nameIfChanged returns name, unless it was the already the value of (*mp)[id], -// in which case it returns the empty string. -func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string { - if name == "" { - return "" - } - if *mp == nil { - *mp = make(map[int]string) - } - if (*mp)[id] == name { - return "" - } - (*mp)[id] = name - return name -} - -func (w *Writer) condOpenGz() (err error) { - if w.gz == nil { - w.gz, err = w.compressor.Writer(w.cw) - if w.gz != nil { - w.gz = w.uncompressedCounter.register(w.gz) - } - } - return -} - -// AppendTar reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -func (w *Writer) AppendTar(r io.Reader) error { - return w.appendTar(r, false) -} - -// AppendTarLossLess reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -// -// The difference of this func with AppendTar is that this writes -// the input tar stream into w without any modification (e.g. to header bytes). -// -// Note that if the input tar stream already contains TOC JSON, this returns -// error because w cannot overwrite the TOC JSON to the one generated by w without -// lossy modification. To avoid this error, if the input stream is known to be stargz/estargz, -// you shoud decompress it and remove TOC JSON in advance. -func (w *Writer) AppendTarLossLess(r io.Reader) error { - return w.appendTar(r, true) -} - -func (w *Writer) appendTar(r io.Reader, lossless bool) error { - var src io.Reader - br := bufio.NewReader(r) - if isGzip(br) { - zr, _ := gzip.NewReader(br) - src = zr - } else { - src = io.Reader(br) - } - dst := currentCompressionWriter{w} - var tw *tar.Writer - if !lossless { - tw = tar.NewWriter(dst) // use tar writer only when this isn't lossless mode. - } - tr := tar.NewReader(src) - if lossless { - tr.RawAccounting = true - } - prevOffset := w.cw.n - var prevOffsetUncompressed int64 - for { - h, err := tr.Next() - if err == io.EOF { - if lossless { - if remain := tr.RawBytes(); len(remain) > 0 { - // Collect the remaining null bytes. - // https://github.com/vbatts/tar-split/blob/80a436fd6164c557b131f7c59ed69bd81af69761/concept/main.go#L49-L53 - if _, err := dst.Write(remain); err != nil { - return err - } - } - } - break - } - if err != nil { - return fmt.Errorf("error reading from source tar: tar.Reader.Next: %v", err) - } - if cleanEntryName(h.Name) == TOCTarName { - // It is possible for a layer to be "stargzified" twice during the - // distribution lifecycle. So we reserve "TOCTarName" here to avoid - // duplicated entries in the resulting layer. - if lossless { - // We cannot handle this in lossless way. - return fmt.Errorf("existing TOC JSON is not allowed; decompress layer before append") - } - continue - } - - xattrs := make(map[string][]byte) - const xattrPAXRecordsPrefix = "SCHILY.xattr." - if h.PAXRecords != nil { - for k, v := range h.PAXRecords { - if strings.HasPrefix(k, xattrPAXRecordsPrefix) { - xattrs[k[len(xattrPAXRecordsPrefix):]] = []byte(v) - } - } - } - ent := &TOCEntry{ - Name: h.Name, - Mode: h.Mode, - UID: h.Uid, - GID: h.Gid, - Uname: w.nameIfChanged(&w.lastUsername, h.Uid, h.Uname), - Gname: w.nameIfChanged(&w.lastGroupname, h.Gid, h.Gname), - ModTime3339: formatModtime(h.ModTime), - Xattrs: xattrs, - } - if err := w.condOpenGz(); err != nil { - return err - } - if tw != nil { - if err := tw.WriteHeader(h); err != nil { - return err - } - } else { - if _, err := dst.Write(tr.RawBytes()); err != nil { - return err - } - } - switch h.Typeflag { - case tar.TypeLink: - ent.Type = "hardlink" - ent.LinkName = h.Linkname - case tar.TypeSymlink: - ent.Type = "symlink" - ent.LinkName = h.Linkname - case tar.TypeDir: - ent.Type = "dir" - case tar.TypeReg: - ent.Type = "reg" - ent.Size = h.Size - case tar.TypeChar: - ent.Type = "char" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeBlock: - ent.Type = "block" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeFifo: - ent.Type = "fifo" - default: - return fmt.Errorf("unsupported input tar entry %q", h.Typeflag) - } - - // We need to keep a reference to the TOC entry for regular files, so that we - // can fill the digest later. - var regFileEntry *TOCEntry - var payloadDigest digest.Digester - if h.Typeflag == tar.TypeReg { - regFileEntry = ent - payloadDigest = digest.Canonical.Digester() - } - - if h.Typeflag == tar.TypeReg && ent.Size > 0 { - var written int64 - totalSize := ent.Size // save it before we destroy ent - tee := io.TeeReader(tr, payloadDigest.Hash()) - for written < totalSize { - chunkSize := int64(w.chunkSize()) - remain := totalSize - written - if remain < chunkSize { - chunkSize = remain - } else { - ent.ChunkSize = chunkSize - } - - // We flush the underlying compression writer here to correctly calculate "w.cw.n". - if err := w.flushGz(); err != nil { - return err - } - if w.needsOpenGz(ent) || w.cw.n-prevOffset >= int64(w.MinChunkSize) { - if err := w.closeGz(); err != nil { - return err - } - ent.Offset = w.cw.n - prevOffset = ent.Offset - prevOffsetUncompressed = w.uncompressedCounter.n - } else { - ent.Offset = prevOffset - ent.InnerOffset = w.uncompressedCounter.n - prevOffsetUncompressed - } - - ent.ChunkOffset = written - chunkDigest := digest.Canonical.Digester() - - if err := w.condOpenGz(); err != nil { - return err - } - - teeChunk := io.TeeReader(tee, chunkDigest.Hash()) - var out io.Writer - if tw != nil { - out = tw - } else { - out = dst - } - if _, err := io.CopyN(out, teeChunk, chunkSize); err != nil { - return fmt.Errorf("error copying %q: %v", h.Name, err) - } - ent.ChunkDigest = chunkDigest.Digest().String() - w.toc.Entries = append(w.toc.Entries, ent) - written += chunkSize - ent = &TOCEntry{ - Name: h.Name, - Type: "chunk", - } - } - } else { - w.toc.Entries = append(w.toc.Entries, ent) - } - if payloadDigest != nil { - regFileEntry.Digest = payloadDigest.Digest().String() - } - if tw != nil { - if err := tw.Flush(); err != nil { - return err - } - } - } - remainDest := io.Discard - if lossless { - remainDest = dst // Preserve the remaining bytes in lossless mode - } - _, err := io.Copy(remainDest, src) - return err -} - -func (w *Writer) needsOpenGz(ent *TOCEntry) bool { - if ent.Type != "reg" { - return false - } - if w.needsOpenGzEntries == nil { - return false - } - _, ok := w.needsOpenGzEntries[ent.Name] - return ok -} - -// DiffID returns the SHA-256 of the uncompressed tar bytes. -// It is only valid to call DiffID after Close. -func (w *Writer) DiffID() string { - return fmt.Sprintf("sha256:%x", w.diffHash.Sum(nil)) -} - -func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) { - for _, d := range decompressors { - if s := d.FooterSize(); res < s && s <= blobSize { - res = s - } - } - return -} - -func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts openOpts) (*Reader, error) { - if tocOff < 0 { - // This means that TOC isn't contained in the blob. - // We pass nil reader to ParseTOC and expect that ParseTOC acquire TOC from - // the external location. - start := time.Now() - toc, tocDgst, err := d.ParseTOC(nil) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - if len(tocBytes) > 0 { - start := time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err == nil { - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - } - - start := time.Now() - tocBytes = make([]byte, tocSize) - if _, err := sr.ReadAt(tocBytes, tocOff); err != nil { - return nil, fmt.Errorf("error reading %d byte TOC targz: %v", len(tocBytes), err) - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - start = time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil -} - -func formatModtime(t time.Time) string { - if t.IsZero() || t.Unix() == 0 { - return "" - } - return t.UTC().Round(time.Second).Format(time.RFC3339) -} - -func cleanEntryName(name string) string { - // Use path.Clean to consistently deal with path separators across platforms. - return strings.TrimPrefix(path.Clean("/"+name), "/") -} - -// countWriter counts how many bytes have been written to its wrapped -// io.Writer. -type countWriter struct { - w io.Writer - n int64 -} - -func (cw *countWriter) Write(p []byte) (n int, err error) { - n, err = cw.w.Write(p) - cw.n += int64(n) - return -} - -type countWriteFlusher struct { - io.WriteCloser - n int64 -} - -func (wc *countWriteFlusher) register(w io.WriteCloser) io.WriteCloser { - wc.WriteCloser = w - return wc -} - -func (wc *countWriteFlusher) Write(p []byte) (n int, err error) { - n, err = wc.WriteCloser.Write(p) - wc.n += int64(n) - return -} - -func (wc *countWriteFlusher) Flush() error { - if f, ok := wc.WriteCloser.(interface { - Flush() error - }); ok { - return f.Flush() - } - return nil -} - -func (wc *countWriteFlusher) Close() error { - err := wc.WriteCloser.Close() - wc.WriteCloser = nil - return err -} - -// isGzip reports whether br is positioned right before an upcoming gzip stream. -// It does not consume any bytes from br. -func isGzip(br *bufio.Reader) bool { - const ( - gzipID1 = 0x1f - gzipID2 = 0x8b - gzipDeflate = 8 - ) - peek, _ := br.Peek(3) - return len(peek) >= 3 && peek[0] == gzipID1 && peek[1] == gzipID2 && peek[2] == gzipDeflate -} - -func positive(n int64) int64 { - if n < 0 { - return 0 - } - return n -} - -type countReader struct { - r io.Reader - n int64 -} - -func (cr *countReader) Read(p []byte) (n int, err error) { - n, err = cr.r.Read(p) - cr.n += int64(n) - return -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go deleted file mode 100644 index 88fa13b19..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ /dev/null @@ -1,237 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "encoding/binary" - "encoding/json" - "fmt" - "hash" - "io" - "strconv" - - digest "github.com/opencontainers/go-digest" -) - -type gzipCompression struct { - *GzipCompressor - *GzipDecompressor -} - -func newGzipCompressionWithLevel(level int) Compression { - return &gzipCompression{ - &GzipCompressor{level}, - &GzipDecompressor{}, - } -} - -func NewGzipCompressor() *GzipCompressor { - return &GzipCompressor{gzip.BestCompression} -} - -func NewGzipCompressorWithLevel(level int) *GzipCompressor { - return &GzipCompressor{level} -} - -type GzipCompressor struct { - compressionLevel int -} - -func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) { - return gzip.NewWriterLevel(w, gc.compressionLevel) -} - -func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) { - tocJSON, err := json.MarshalIndent(toc, "", "\t") - if err != nil { - return "", err - } - gz, _ := gzip.NewWriterLevel(w, gc.compressionLevel) - gw := io.Writer(gz) - if diffHash != nil { - gw = io.MultiWriter(gz, diffHash) - } - tw := tar.NewWriter(gw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - return "", err - } - if _, err := tw.Write(tocJSON); err != nil { - return "", err - } - - if err := tw.Close(); err != nil { - return "", err - } - if err := gz.Close(); err != nil { - return "", err - } - if _, err := w.Write(gzipFooterBytes(off)); err != nil { - return "", err - } - return digest.FromBytes(tocJSON), nil -} - -// gzipFooterBytes returns the 51 bytes footer. -func gzipFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - - // Extra header indicating the offset of TOCJSON - // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 - header := make([]byte, 4) - header[0], header[1] = 'S', 'G' - subfield := fmt.Sprintf("%016xSTARGZ", tocOff) - binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) - } - return buf.Bytes() -} - -type GzipDecompressor struct{} - -func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != FooterSize { - return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, err - } - defer zr.Close() - extra := zr.Extra - si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] - if si1 != 'S' || si2 != 'G' { - return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) - } - if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { - return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) - } - if string(subfield[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") - } - tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *GzipDecompressor) FooterSize() int64 { - return FooterSize -} - -func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -type LegacyGzipDecompressor struct{} - -func (gz *LegacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *LegacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != legacyFooterSize { - return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to get footer gzip reader: %w", err) - } - defer zr.Close() - extra := zr.Extra - if len(extra) != 16+len("STARGZ") { - return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") - } - if string(extra[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") - } - tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *LegacyGzipDecompressor) FooterSize() int64 { - return legacyFooterSize -} - -func (gz *LegacyGzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - tr, err := decompressTOCEStargz(r) - if err != nil { - return nil, "", err - } - dgstr := digest.Canonical.Digester() - toc = new(JTOC) - if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil { - return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err) - } - if err := tr.Close(); err != nil { - return nil, "", err - } - return toc, dgstr.Digest(), nil -} - -func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) { - zr, err := gzip.NewReader(r) - if err != nil { - return nil, fmt.Errorf("malformed TOC gzip header: %v", err) - } - zr.Multistream(false) - tr := tar.NewReader(zr) - h, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err) - } - if h.Name != TOCTarName { - return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName) - } - return readCloser{tr, zr.Close}, nil -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go deleted file mode 100644 index ff165e090..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ /dev/null @@ -1,2403 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto/rand" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "os" - "path/filepath" - "reflect" - "sort" - "strings" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" -) - -// TestingController is Compression with some helper methods necessary for testing. -type TestingController interface { - Compression - TestStreams(t TestingT, b []byte, streams []int64) - DiffIDOf(TestingT, []byte) string - String() string -} - -// TestingT is the minimal set of testing.T required to run the -// tests defined in CompressionTestSuite. This interface exists to prevent -// leaking the testing package from being exposed outside tests. -type TestingT interface { - Errorf(format string, args ...any) - FailNow() - Failed() bool - Fatal(args ...any) - Fatalf(format string, args ...any) - Logf(format string, args ...any) - Parallel() -} - -// Runner allows running subtests of TestingT. This exists instead of adding -// a Run method to TestingT interface because the Run implementation of -// testing.T would not satisfy the interface. -type Runner func(t TestingT, name string, fn func(t TestingT)) - -type TestRunner struct { - TestingT - Runner Runner -} - -func (r *TestRunner) Run(name string, run func(*TestRunner)) { - r.Runner(r.TestingT, name, func(t TestingT) { - run(&TestRunner{TestingT: t, Runner: r.Runner}) - }) -} - -// CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them. -func CompressionTestSuite(t *TestRunner, controllers ...TestingControllerFactory) { - t.Run("testBuild", func(t *TestRunner) { t.Parallel(); testBuild(t, controllers...) }) - t.Run("testDigestAndVerify", func(t *TestRunner) { - t.Parallel() - testDigestAndVerify(t, controllers...) - }) - t.Run("testWriteAndOpen", func(t *TestRunner) { t.Parallel(); testWriteAndOpen(t, controllers...) }) -} - -type TestingControllerFactory func() TestingController - -const ( - uncompressedType int = iota - gzipType - zstdType -) - -var srcCompressions = []int{ - uncompressedType, - gzipType, - zstdType, -} - -var allowedPrefix = [4]string{"", "./", "/", "../"} - -// testBuild tests the resulting stargz blob built by this pkg has the same -// contents as the normal stargz blob. -func testBuild(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - chunkSize int - minChunkSize []int - in []tarEntry - }{ - { - name: "regfiles and directories", - chunkSize: 4, - in: tarOf( - file("foo", "test1"), - dir("foo2/"), - file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})), - ), - }, - { - name: "empty files", - chunkSize: 4, - in: tarOf( - file("foo", "tttttt"), - file("foo_empty", ""), - file("foo2", "tttttt"), - file("foo_empty2", ""), - file("foo3", "tttttt"), - file("foo_empty3", ""), - file("foo4", "tttttt"), - file("foo_empty4", ""), - file("foo5", "tttttt"), - file("foo_empty5", ""), - file("foo6", "tttttt"), - ), - }, - { - name: "various files", - chunkSize: 4, - minChunkSize: []int{0, 64000}, - in: tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo1.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - { - name: "no contents", - chunkSize: 4, - in: tarOf( - file("baz.txt", ""), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - } - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, prefix := range allowedPrefix { - prefix := prefix - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) { - tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) - // Test divideEntries() - entries, err := sortEntries(tarBlob, nil, nil) // identical order - if err != nil { - t.Fatalf("failed to parse tar: %v", err) - } - var merged []*entry - for _, part := range divideEntries(entries, 4) { - merged = append(merged, part...) - } - if !reflect.DeepEqual(entries, merged) { - for _, e := range entries { - t.Logf("Original: %v", e.header) - } - for _, e := range merged { - t.Logf("Merged: %v", e.header) - } - t.Errorf("divided entries couldn't be merged") - return - } - - // Prepare sample data - cl1 := newCL() - wantBuf := new(bytes.Buffer) - sw := NewWriterWithCompressor(wantBuf, cl1) - sw.MinChunkSize = minChunkSize - sw.ChunkSize = tt.chunkSize - if err := sw.AppendTar(tarBlob); err != nil { - t.Fatalf("failed to append tar to want stargz: %v", err) - } - if _, err := sw.Close(); err != nil { - t.Fatalf("failed to prepare want stargz: %v", err) - } - wantData := wantBuf.Bytes() - want, err := Open(io.NewSectionReader( - bytes.NewReader(wantData), 0, int64(len(wantData))), - WithDecompressors(cl1), - ) - if err != nil { - t.Fatalf("failed to parse the want stargz: %v", err) - } - - // Prepare testing data - var opts []Option - if minChunkSize > 0 { - opts = append(opts, WithMinChunkSize(minChunkSize)) - } - cl2 := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - append(opts, WithChunkSize(tt.chunkSize), WithCompression(cl2))...) - if err != nil { - t.Fatalf("failed to build stargz: %v", err) - } - defer rc.Close() - gotBuf := new(bytes.Buffer) - if _, err := io.Copy(gotBuf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - gotData := gotBuf.Bytes() - got, err := Open(io.NewSectionReader( - bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), - WithDecompressors(cl2), - ) - if err != nil { - t.Fatalf("failed to parse the got stargz: %v", err) - } - - // Check DiffID is properly calculated - rc.Close() - diffID := rc.DiffID() - wantDiffID := cl2.DiffIDOf(t, gotData) - if diffID.String() != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - // Compare as stargz - if !isSameVersion(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz hasn't same json") - return - } - if !isSameEntries(t, want, got) { - t.Errorf("built stargz isn't same as the original") - return - } - - // Compare as tar.gz - if !isSameTarGz(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz isn't same tar.gz") - return - } - }) - } - } - } - } - } - } -} - -func isSameTarGz(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aGz, err := cla.Reader(bytes.NewReader(a)) - if err != nil { - t.Fatalf("failed to read A") - } - defer aGz.Close() - bGz, err := clb.Reader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("failed to read B") - } - defer bGz.Close() - - // Same as tar's Next() method but ignores landmarks and TOCJSON file - next := func(r *tar.Reader) (h *tar.Header, err error) { - for { - if h, err = r.Next(); err != nil { - return - } - if h.Name != PrefetchLandmark && - h.Name != NoPrefetchLandmark && - h.Name != TOCTarName { - return - } - } - } - - aTar := tar.NewReader(aGz) - bTar := tar.NewReader(bGz) - for { - // Fetch and parse next header. - aH, aErr := next(aTar) - bH, bErr := next(bTar) - if aErr != nil || bErr != nil { - if aErr == io.EOF && bErr == io.EOF { - break - } - t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr) - } - if !reflect.DeepEqual(aH, bH) { - t.Logf("different header (A = %v; B = %v)", aH, bH) - return false - - } - aFile, err := io.ReadAll(aTar) - if err != nil { - t.Fatal("failed to read tar payload of A") - } - bFile, err := io.ReadAll(bTar) - if err != nil { - t.Fatal("failed to read tar payload of B") - } - if !bytes.Equal(aFile, bFile) { - t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b)) - return false - } - } - - return true -} - -func isSameVersion(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), cla) - if err != nil { - t.Fatalf("failed to parse A: %v", err) - } - bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), clb) - if err != nil { - t.Fatalf("failed to parse B: %v", err) - } - t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, aJTOC)) - t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bJTOC)) - return aJTOC.Version == bJTOC.Version -} - -func isSameEntries(t TestingT, a, b *Reader) bool { - aroot, ok := a.Lookup("") - if !ok { - t.Fatalf("failed to get root of A") - } - broot, ok := b.Lookup("") - if !ok { - t.Fatalf("failed to get root of B") - } - aEntry := stargzEntry{aroot, a} - bEntry := stargzEntry{broot, b} - return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry) -} - -func compressBlob(t TestingT, src *io.SectionReader, srcCompression int) *io.SectionReader { - buf := new(bytes.Buffer) - var w io.WriteCloser - var err error - switch srcCompression { - case gzipType: - w = gzip.NewWriter(buf) - case zstdType: - w, err = zstd.NewWriter(buf) - if err != nil { - t.Fatalf("failed to init zstd writer: %v", err) - } - default: - return src - } - src.Seek(0, io.SeekStart) - if _, err := io.Copy(w, src); err != nil { - t.Fatalf("failed to compress source") - } - if err := w.Close(); err != nil { - t.Fatalf("failed to finalize compress source") - } - data := buf.Bytes() - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) - -} - -type stargzEntry struct { - e *TOCEntry - r *Reader -} - -// contains checks if all child entries in "b" are also contained in "a". -// This function also checks if the files/chunks contain the same contents among "a" and "b". -func contains(t TestingT, a, b stargzEntry) bool { - ae, ar := a.e, a.r - be, br := b.e, b.r - t.Logf("Comparing: %q vs %q", ae.Name, be.Name) - if !equalEntry(ae, be) { - t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be) - return false - } - if ae.Type == "dir" { - t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name, - allChildrenName(ae), allChildrenName(be)) - iscontain := true - ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool { - // Walk through all files on this stargz file. - - if aChild.Name == PrefetchLandmark || - aChild.Name == NoPrefetchLandmark { - return true // Ignore landmarks - } - - // Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory - // because this points to the root directory itself. - if aChild.Name == "" && ae.Name == "" { - return true - } - - bChild, ok := be.LookupChild(aBaseName) - if !ok { - t.Logf("%q (base: %q): not found in b: %v", - ae.Name, aBaseName, allChildrenName(be)) - iscontain = false - return false - } - - childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r}) - if !childcontain { - t.Logf("%q != %q: non-equal dir", ae.Name, be.Name) - iscontain = false - return false - } - return true - }) - return iscontain - } else if ae.Type == "reg" { - af, err := ar.OpenFile(ae.Name) - if err != nil { - t.Fatalf("failed to open file %q on A: %v", ae.Name, err) - } - bf, err := br.OpenFile(be.Name) - if err != nil { - t.Fatalf("failed to open file %q on B: %v", be.Name, err) - } - - var nr int64 - for nr < ae.Size { - abytes, anext, aok := readOffset(t, af, nr, a) - bbytes, bnext, bok := readOffset(t, bf, nr, b) - if !aok && !bok { - break - } else if !aok || !bok || anext != bnext { - t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v", - ae.Name, be.Name, nr, aok, bok, anext, bnext) - return false - } - nr = anext - if !bytes.Equal(abytes, bbytes) { - t.Logf("%q != %q: different contents %v vs %v", - ae.Name, be.Name, string(abytes), string(bbytes)) - return false - } - } - return true - } - - return true -} - -func allChildrenName(e *TOCEntry) (children []string) { - e.ForeachChild(func(baseName string, _ *TOCEntry) bool { - children = append(children, baseName) - return true - }) - return -} - -func equalEntry(a, b *TOCEntry) bool { - // Here, we selectively compare fileds that we are interested in. - return a.Name == b.Name && - a.Type == b.Type && - a.Size == b.Size && - a.ModTime3339 == b.ModTime3339 && - a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime time.Time - a.LinkName == b.LinkName && - a.Mode == b.Mode && - a.UID == b.UID && - a.GID == b.GID && - a.Uname == b.Uname && - a.Gname == b.Gname && - (a.Offset >= 0) == (b.Offset >= 0) && - (a.NextOffset() > 0) == (b.NextOffset() > 0) && - a.DevMajor == b.DevMajor && - a.DevMinor == b.DevMinor && - a.NumLink == b.NumLink && - reflect.DeepEqual(a.Xattrs, b.Xattrs) && - // chunk-related infomations aren't compared in this function. - // ChunkOffset int64 `json:"chunkOffset,omitempty"` - // ChunkSize int64 `json:"chunkSize,omitempty"` - // children map[string]*TOCEntry - a.Digest == b.Digest -} - -func readOffset(t TestingT, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) { - ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset) - if !ok { - return nil, 0, false - } - data := make([]byte, ce.ChunkSize) - t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset()) - n, err := r.ReadAt(data, ce.ChunkOffset) - if err != nil { - t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v", - e.e.Name, ce.ChunkOffset, ce.ChunkSize, err) - } - if int64(n) != ce.ChunkSize { - t.Fatalf("unexpected copied data size %d; want %d", - n, ce.ChunkSize) - } - return data[:n], offset + ce.ChunkSize, true -} - -func dumpTOCJSON(t TestingT, tocJSON *JTOC) string { - jtocData, err := json.Marshal(*tocJSON) - if err != nil { - t.Fatalf("failed to marshal TOC JSON: %v", err) - } - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil { - t.Fatalf("failed to read toc json blob: %v", err) - } - return buf.String() -} - -const chunkSize = 3 - -type check func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) - -// testDigestAndVerify runs specified checks against sample stargz blobs. -func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - tarInit func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) - checks []check - minChunkSize []int - }{ - { - name: "no-regfile", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - dir("test/"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - dir("test2/"), // modified - ), allowedPrefix[0])), - }, - }, - { - name: "small-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "bbb", dgstMap), - ) - }, - minChunkSize: []int{0, 64000}, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", ""), - file("foo.txt", "M"), // modified - dir("test/"), - file("test/bar.txt", "bbb"), - ), allowedPrefix[0])), - // checkVerifyInvalidTOCEntryFail("foo.txt"), // TODO - checkVerifyBrokenContentFail("foo.txt"), - }, - }, - { - name: "big-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified - file("foo.txt", "a"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - { - name: "with-non-regfiles", - minChunkSize: []int{0, 64000}, - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - regDigest(t, "bar/foo2.txt", "b", dgstMap), - regDigest(t, "foo3.txt", "c", dgstMap), - symlink("barlink", "test/bar.txt"), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - dir("test2/"), - link("test2/bazlink", "foo.txt"), // modified - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - } - - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) { - // Get original tar file and chunk digests - dgstMap := make(map[string]digest.Digest) - tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat) - - cl := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - tocDigest := rc.TOCDigest() - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - newStargz := buf.Bytes() - // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. - dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) - - for _, check := range tt.checks { - check(t, newStargz, tocDigest, dgstMap, cl, newCL) - } - }) - } - } - } - } - } - } -} - -// checkStargzTOC checks the TOC JSON of the passed stargz has the expected -// digest and contains valid chunks. It walks all entries in the stargz and -// checks all chunk digests stored to the TOC JSON match the actual contents. -func checkStargzTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - digestMapTOC, err := listDigests(io.NewSectionReader( - bytes.NewReader(sgzData), 0, int64(len(sgzData))), - controller, - ) - if err != nil { - t.Fatalf("failed to list digest: %v", err) - } - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - // Check the digest of TOC JSON based on the actual contents - // It's sure that TOC JSON exists in this archive because - // Open succeeded. - dgstr := digest.Canonical.Digester() - if _, err := io.Copy(dgstr.Hash(), tr); err != nil { - t.Fatalf("failed to calculate digest of TOC JSON: %v", - err) - } - if dgstr.Digest() != tocDigest { - t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest()) - } - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - // Get the original digest to make sure the file contents are kept unchanged - // from the original tar, during the whole conversion steps. - id := chunkID(h.Name, n, ce.ChunkSize) - want, ok := dgstMap[id] - if !ok { - t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v", - h.Name, n, ce.ChunkSize, dgstMap) - return - } - found[id] = true - - // Check the file contents - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if want != dgstr.Digest() { - t.Errorf("Invalid contents in converted stargz %q: %q; want %q", - h.Name, dgstr.Digest(), want) - return - } - - // Check the digest stored in TOC JSON - dgstTOC, ok := digestMapTOC[ce.Offset] - if !ok { - t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered", - h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset) - } - if want != dgstTOC { - t.Errorf("Invalid digest in TOCEntry %q: %q; want %q", - h.Name, dgstTOC, want) - return - } - - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyTOC checks the verification works for the TOC JSON of the passed -// stargz. It walks all entries in the stargz and checks the verifications for -// all chunks work. -func checkVerifyTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Errorf("failed to verify stargz: %v", err) - return - } - - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - v, err := ev.Verifier(ce) - if err != nil { - t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n) - } - - found[chunkID(h.Name, n, ce.ChunkSize)] = true - - // Check the file contents - if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if !v.Verified() { - t.Errorf("Invalid contents in converted stargz %q (should be succeeded)", - h.Name) - return - } - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be -// detected during the verification and the verification returns an error. -func checkVerifyInvalidTOCEntryFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - funcs := map[string]rewriteFunc{ - "lost digest in a entry": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var found bool - for _, e := range toc.Entries { - if cleanEntryName(e.Name) == filename { - if e.Type != "reg" && e.Type != "chunk" { - t.Fatalf("entry %q to break must be regfile or chunk", filename) - } - if e.ChunkDigest == "" { - t.Fatalf("entry %q is already invalid", filename) - } - e.ChunkDigest = "" - found = true - } - } - if !found { - t.Fatalf("rewrite target not found") - } - }, - "duplicated entry offset": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var ( - sampleEntry *TOCEntry - targetEntry *TOCEntry - ) - for _, e := range toc.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if cleanEntryName(e.Name) == filename { - targetEntry = e - } else { - sampleEntry = e - } - } - } - if sampleEntry == nil { - t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") - return - } - if targetEntry == nil { - t.Fatalf("rewrite target not found") - return - } - targetEntry.Offset = sampleEntry.Offset - }, - } - - for name, rFunc := range funcs { - t.Run(name, func(t *TestRunner) { - newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, controller) - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, newSgz); err != nil { - t.Fatalf("failed to get converted stargz") - } - isgz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(newTocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - }) - } - } -} - -// checkVerifyInvalidStargzFail checks if the verification detects that the -// given stargz file doesn't match to the expected digest and returns error. -func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - cl := newController() - rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - mStargz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))), - WithDecompressors(cl), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(tocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - } -} - -// checkVerifyBrokenContentFail checks if the verifier detects broken contents -// that doesn't match to the expected digest and returns error. -func checkVerifyBrokenContentFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - // Parse stargz file - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Fatalf("failed to verify stargz: %v", err) - return - } - - // Open the target file - sr, err := sgz.OpenFile(filename) - if err != nil { - t.Fatalf("failed to open file %q", filename) - } - ce, ok := sgz.ChunkEntryForOffset(filename, 0) - if !ok { - t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0) - return - } - if ce.ChunkSize == 0 { - t.Fatalf("file mustn't be empty") - return - } - data := make([]byte, ce.ChunkSize) - if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil { - t.Errorf("failed to get data of a chunk of %q(offset=%q)", - filename, ce.ChunkOffset) - } - - // Check the broken chunk (must fail) - v, err := ev.Verifier(ce) - if err != nil { - t.Fatalf("failed to get verifier for %q", filename) - } - broken := append([]byte{^data[0]}, data[1:]...) - if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - filename, ce.ChunkOffset, ce.ChunkSize) - } - if v.Verified() { - t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)", - filename, data, broken) - } - } -} - -func chunkID(name string, offset, size int64) string { - return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size) -} - -type rewriteFunc func(t TestingT, toc *JTOC, sgz *io.SectionReader) - -func rewriteTOCJSON(t TestingT, sgz *io.SectionReader, rewrite rewriteFunc, controller TestingController) (newSgz io.Reader, tocDigest digest.Digest) { - decodedJTOC, jtocOffset, err := parseStargz(sgz, controller) - if err != nil { - t.Fatalf("failed to extract TOC JSON: %v", err) - } - - rewrite(t, decodedJTOC, sgz) - - tocFooter, tocDigest, err := tocAndFooter(controller, decodedJTOC, jtocOffset) - if err != nil { - t.Fatalf("failed to create toc and footer: %v", err) - } - - // Reconstruct stargz file with the modified TOC JSON - if _, err := sgz.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to reset the seek position of stargz: %v", err) - } - return io.MultiReader( - io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON) - tocFooter, // Rewritten TOC and footer - ), tocDigest -} - -func listDigests(sgz *io.SectionReader, controller TestingController) (map[int64]digest.Digest, error) { - decodedJTOC, _, err := parseStargz(sgz, controller) - if err != nil { - return nil, err - } - digestMap := make(map[int64]digest.Digest) - for _, e := range decodedJTOC.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if e.Type == "reg" && e.Size == 0 { - continue // ignores empty file - } - if e.ChunkDigest == "" { - return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", - e.Name, e.Offset) - } - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, err - } - digestMap[e.Offset] = d - } - } - return digestMap, nil -} - -func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJTOC *JTOC, jtocOffset int64, err error) { - fSize := controller.FooterSize() - footer := make([]byte, fSize) - if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil { - return nil, 0, fmt.Errorf("error reading footer: %w", err) - } - _, tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse footer: %w", err) - } - - // Decode the TOC JSON - var tocReader io.Reader - if tocOffset >= 0 { - tocReader = io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize) - } - decodedJTOC, _, err = controller.ParseTOC(tocReader) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse TOC: %w", err) - } - return decodedJTOC, tocOffset, nil -} - -func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { - const content = "Some contents" - invalidUtf8 := "\xff\xfe\xfd" - - xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8} - sampleOwner := owner{uid: 50, gid: 100} - - data64KB := randomContents(64000) - - tests := []struct { - name string - chunkSize int - minChunkSize int - in []tarEntry - want []stargzCheck - wantNumGz int // expected number of streams - - wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz - wantFailOnLossLess bool - wantTOCVersion int // default = 1 - }{ - { - name: "empty", - in: tarOf(), - wantNumGz: 2, // (empty tar) + TOC + footer - want: checks( - numTOCEntries(0), - ), - }, - { - name: "1dir_1empty_file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", ""), - ), - wantNumGz: 3, // dir, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", 0), - entryHasChildren("foo", "bar.txt"), - hasFileDigest("foo/bar.txt", digestFor("")), - ), - }, - { - name: "1dir_1file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", content, xAttrFile), - ), - wantNumGz: 4, // var dir, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - hasFileDigest("foo/bar.txt", digestFor(content)), - hasFileContentsRange("foo/bar.txt", 0, content), - hasFileContentsRange("foo/bar.txt", 1, content[1:]), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar.txt"), - hasFileXattrs("foo/bar.txt", "foo", "bar"), - hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8), - ), - }, - { - name: "2meta_2file", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - }, - { - name: "3dir", - in: tarOf( - dir("bar/"), - dir("foo/"), - dir("foo/bar/"), - ), - wantNumGz: 3, // 3 dirs, TOC, footer - want: checks( - hasDirLinkCount("bar/", 2), - hasDirLinkCount("foo/", 3), - hasDirLinkCount("foo/bar/", 2), - ), - }, - { - name: "symlink", - in: tarOf( - dir("foo/"), - symlink("foo/bar", "../../x"), - ), - wantNumGz: 3, // metas + TOC + footer - want: checks( - numTOCEntries(2), - hasSymlink("foo/bar", "../../x"), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar"), - ), - }, - { - name: "chunked_file", - chunkSize: 4, - in: tarOf( - dir("foo/"), - file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"), - ), - wantNumGz: 9, // dir + big.txt(6 chunks) + TOC + footer - want: checks( - numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file - hasDir("foo/"), - hasFileLen("foo/big.txt", len("This is such a big file")), - hasFileDigest("foo/big.txt", digestFor("This is such a big file")), - hasFileContentsRange("foo/big.txt", 0, "This is such a big file"), - hasFileContentsRange("foo/big.txt", 1, "his is such a big file"), - hasFileContentsRange("foo/big.txt", 2, "is is such a big file"), - hasFileContentsRange("foo/big.txt", 3, "s is such a big file"), - hasFileContentsRange("foo/big.txt", 4, " is such a big file"), - hasFileContentsRange("foo/big.txt", 5, "is such a big file"), - hasFileContentsRange("foo/big.txt", 6, "s such a big file"), - hasFileContentsRange("foo/big.txt", 7, " such a big file"), - hasFileContentsRange("foo/big.txt", 8, "such a big file"), - hasFileContentsRange("foo/big.txt", 9, "uch a big file"), - hasFileContentsRange("foo/big.txt", 10, "ch a big file"), - hasFileContentsRange("foo/big.txt", 11, "h a big file"), - hasFileContentsRange("foo/big.txt", 12, " a big file"), - hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""), - hasChunkEntries("foo/big.txt", 6), - ), - }, - { - name: "recursive", - in: tarOf( - dir("/", sampleOwner), - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // dirs, bar.txt alone, TOC, footer - want: checks( - maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt" - ), - }, - { - name: "block_char_fifo", - in: tarOf( - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "b", - Typeflag: tar.TypeBlock, - Devmajor: 123, - Devminor: 456, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "c", - Typeflag: tar.TypeChar, - Devmajor: 111, - Devminor: 222, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "f", - Typeflag: tar.TypeFifo, - Format: format, - }) - }), - ), - wantNumGz: 3, - want: checks( - lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}), - lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}), - lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}), - ), - }, - { - name: "modes", - in: tarOf( - dir("foo1/", 0755|os.ModeDir|os.ModeSetgid), - file("foo1/bar1", content, 0700|os.ModeSetuid), - file("foo1/bar2", content, 0755|os.ModeSetgid), - dir("foo2/", 0755|os.ModeDir|os.ModeSticky), - file("foo2/bar3", content, 0755|os.ModeSticky), - dir("foo3/", 0755|os.ModeDir), - file("foo3/bar4", content, os.FileMode(0700)), - file("foo3/bar5", content, os.FileMode(0755)), - ), - wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer - want: checks( - hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid), - hasMode("foo1/bar1", 0700|os.ModeSetuid), - hasMode("foo1/bar2", 0755|os.ModeSetgid), - hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky), - hasMode("foo2/bar3", 0755|os.ModeSticky), - hasMode("foo3/", 0755|os.ModeDir), - hasMode("foo3/bar4", os.FileMode(0700)), - hasMode("foo3/bar5", os.FileMode(0755)), - ), - }, - { - name: "lossy", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - file(TOCTarName, "dummy"), // ignored by the writer. (lossless write returns error) - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - wantFailOnLossLess: true, - }, - { - name: "hardlink should be replaced to the destination entry", - in: tarOf( - dir("foo/"), - file("foo/foo1", "test"), - link("foolink", "foo/foo1"), - ), - wantNumGz: 4, // dir, foo1 + link, TOC, footer - want: checks( - mustSameEntry("foo/foo1", "foolink"), - ), - }, - { - name: "several_files_in_chunk", - minChunkSize: 8000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - file("foo22", "ccc"), - dir("bar/"), - file("bar/bar.txt", "aaa"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed "data64KB" is still larger than 8KB - wantNumGz: 4, // dir+foo1, foo2+foo22+dir+bar.txt+foo3, TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1, foo2, foo22, dir, bar.txt, foo3 - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo22", len("ccc")), - hasFileLen("bar/bar.txt", len("aaa")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo22", digestFor("ccc")), - hasFileDigest("bar/bar.txt", digestFor("aaa")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo22", 0, "ccc", chunkInfo{"foo2", "bb"}, chunkInfo{"bar/bar.txt", "aaa"}, chunkInfo{"foo3", data64KB}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo22", 0, "ccc"), - hasFileContentsRange("foo22", 1, "cc"), - hasFileContentsRange("foo22", 2, "c"), - hasFileContentsRange("bar/bar.txt", 0, "aaa"), - hasFileContentsRange("bar/bar.txt", 1, "aa"), - hasFileContentsRange("bar/bar.txt", 2, "a"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - { - name: "several_files_in_chunk_chunked", - minChunkSize: 8000, - chunkSize: 32000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - dir("bar/"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed chunk of "data64KB" is still larger than 8KB - wantNumGz: 6, // dir+foo1(1), foo1(2), foo2+dir+foo3(1), foo3(2), TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1(2 chunks), foo2, dir, foo3(2 chunks) - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo2", 0, "bb", chunkInfo{"foo3", data64KB[:32000]}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo/foo1", 1, data64KB[1:]), - hasFileContentsRange("foo/foo1", 2, data64KB[2:]), - hasFileContentsRange("foo/foo1", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo/foo1", len(data64KB)-1, data64KB[len(data64KB)-1:]), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - } - - for _, tt := range tests { - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, lossless := range []bool{true, false} { - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) { - var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) - origTarDgstr := digest.Canonical.Digester() - tr = io.TeeReader(tr, origTarDgstr.Hash()) - var stargzBuf bytes.Buffer - cl1 := newCL() - w := NewWriterWithCompressor(&stargzBuf, cl1) - w.ChunkSize = tt.chunkSize - w.MinChunkSize = tt.minChunkSize - if lossless { - err := w.AppendTarLossLess(tr) - if tt.wantFailOnLossLess { - if err != nil { - return // expected to fail - } - t.Fatalf("Append wanted to fail on lossless") - } - if err != nil { - t.Fatalf("Append(lossless): %v", err) - } - } else { - if err := w.AppendTar(tr); err != nil { - t.Fatalf("Append: %v", err) - } - } - if _, err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - b := stargzBuf.Bytes() - - if lossless { - // Check if the result blob reserves original tar metadata - rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl1) - if err != nil { - t.Errorf("failed to decompress blob: %v", err) - return - } - defer rc.Close() - resultDgstr := digest.Canonical.Digester() - if _, err := io.Copy(resultDgstr.Hash(), rc); err != nil { - t.Errorf("failed to read result decompressed blob: %v", err) - return - } - if resultDgstr.Digest() != origTarDgstr.Digest() { - t.Errorf("lossy compression occurred: digest=%v; want %v", - resultDgstr.Digest(), origTarDgstr.Digest()) - return - } - } - - diffID := w.DiffID() - wantDiffID := cl1.DiffIDOf(t, b) - if diffID != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - telemetry, checkCalled := newCalledTelemetry() - sr := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))) - r, err := Open( - sr, - WithDecompressors(cl1), - WithTelemetry(telemetry), - ) - if err != nil { - t.Fatalf("stargz.Open: %v", err) - } - if _, ok := r.Lookup(""); !ok { - t.Fatalf("failed to lookup rootdir: %v", err) - } - wantTOCVersion := 1 - if tt.wantTOCVersion > 0 { - wantTOCVersion = tt.wantTOCVersion - } - if r.toc.Version != wantTOCVersion { - t.Fatalf("invalid TOC Version %d; wanted %d", r.toc.Version, wantTOCVersion) - } - - footerSize := cl1.FooterSize() - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - t.Errorf("failed to read footer: %v", err) - } - _, tocOffset, _, err := cl1.ParseFooter(footer) - if err != nil { - t.Errorf("failed to parse footer: %v", err) - } - if err := checkCalled(tocOffset >= 0); err != nil { - t.Errorf("telemetry failure: %v", err) - } - - wantNumGz := tt.wantNumGz - if lossless && tt.wantNumGzLossLess > 0 { - wantNumGz = tt.wantNumGzLossLess - } - streamOffsets := []int64{0} - prevOffset := int64(-1) - streams := 0 - for _, e := range r.toc.Entries { - if e.Offset > prevOffset { - streamOffsets = append(streamOffsets, e.Offset) - prevOffset = e.Offset - streams++ - } - } - streams++ // TOC - if tocOffset >= 0 { - // toc is in the blob - streamOffsets = append(streamOffsets, tocOffset) - } - streams++ // footer - streamOffsets = append(streamOffsets, footerOffset) - if streams != wantNumGz { - t.Errorf("number of streams in TOC = %d; want %d", streams, wantNumGz) - } - - t.Logf("testing streams: %+v", streamOffsets) - cl1.TestStreams(t, b, streamOffsets) - - for _, want := range tt.want { - want.check(t, r) - } - }) - } - } - } - } - } -} - -type chunkInfo struct { - name string - data string -} - -func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) error) { - var getFooterLatencyCalled bool - var getTocLatencyCalled bool - var deserializeTocLatencyCalled bool - return &Telemetry{ - func(time.Time) { getFooterLatencyCalled = true }, - func(time.Time) { getTocLatencyCalled = true }, - func(time.Time) { deserializeTocLatencyCalled = true }, - }, func(needsGetTOC bool) error { - var allErr []error - if !getFooterLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called")) - } - if needsGetTOC { - if !getTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called")) - } - } - if !deserializeTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) - } - return errorutil.Aggregate(allErr) - } -} - -func digestFor(content string) string { - sum := sha256.Sum256([]byte(content)) - return fmt.Sprintf("sha256:%x", sum) -} - -type numTOCEntries int - -func (n numTOCEntries) check(t TestingT, r *Reader) { - if r.toc == nil { - t.Fatal("nil TOC") - } - if got, want := len(r.toc.Entries), int(n); got != want { - t.Errorf("got %d TOC entries; want %d", got, want) - } - t.Logf("got TOC entries:") - for i, ent := range r.toc.Entries { - entj, _ := json.Marshal(ent) - t.Logf(" [%d]: %s\n", i, entj) - } - if t.Failed() { - t.FailNow() - } -} - -func checks(s ...stargzCheck) []stargzCheck { return s } - -type stargzCheck interface { - check(t TestingT, r *Reader) -} - -type stargzCheckFn func(TestingT, *Reader) - -func (f stargzCheckFn) check(t TestingT, r *Reader) { f(t, r) } - -func maxDepth(max int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup("") - if !ok { - t.Fatal("root directory not found") - } - d, err := getMaxDepth(t, e, 0, 10*max) - if err != nil { - t.Errorf("failed to get max depth (wanted %d): %v", max, err) - return - } - if d != max { - t.Errorf("invalid depth %d; want %d", d, max) - return - } - }) -} - -func getMaxDepth(t TestingT, e *TOCEntry, current, limit int) (max int, rErr error) { - if current > limit { - return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d", - current, limit) - } - max = current - e.ForeachChild(func(baseName string, ent *TOCEntry) bool { - t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name) - d, err := getMaxDepth(t, ent, current+1, limit) - if err != nil { - rErr = err - return false - } - if d > max { - max = d - } - return true - }) - return -} - -func hasFileLen(file string, wantLen int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } else if ent.Size != int64(wantLen) { - t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen) - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileXattrs(file, name, value string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } - if ent.Xattrs == nil { - t.Errorf("file %q has no xattrs", file) - return - } - valueFound, found := ent.Xattrs[name] - if !found { - t.Errorf("file %q has no xattr %q", file, name) - return - } - if string(valueFound) != value { - t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value) - } - - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileDigest(file string, digest string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("didn't find TOCEntry for file %q", file) - } - if ent.Digest != digest { - t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest) - } - }) -} - -func hasFileContentsWithPreRead(file string, offset int, want string, extra ...chunkInfo) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - extraMap := make(map[string]chunkInfo) - for _, e := range extra { - extraMap[e.name] = e - } - var extraNames []string - for n := range extraMap { - extraNames = append(extraNames, n) - } - f, err := r.OpenFileWithPreReader(file, func(e *TOCEntry, cr io.Reader) error { - t.Logf("On %q: got preread of %q", file, e.Name) - ex, ok := extraMap[e.Name] - if !ok { - t.Fatalf("fail on %q: unexpected entry %q: %+v, %+v", file, e.Name, e, extraNames) - } - got, err := io.ReadAll(cr) - if err != nil { - t.Fatalf("fail on %q: failed to read %q: %v", file, e.Name, err) - } - if ex.data != string(got) { - t.Fatalf("fail on %q: unexpected contents of %q: len=%d; want=%d", file, e.Name, len(got), len(ex.data)) - } - delete(extraMap, e.Name) - return nil - }) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d, size %d) = %v, %v", len(got), offset, f.Size(), n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - if len(extraMap) != 0 { - var exNames []string - for _, ex := range extraMap { - exNames = append(exNames, ex.name) - } - t.Fatalf("fail on %q: some entries aren't read: %+v", file, exNames) - } - }) -} - -func hasFileContentsRange(file string, offset int, want string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - f, err := r.OpenFile(file) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - }) -} - -func hasChunkEntries(file string, wantChunks int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("no file for %q", file) - } - if ent.Type != "reg" { - t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type) - } - chunks := r.getChunks(ent) - if len(chunks) != wantChunks { - t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks) - return - } - f := chunks[0] - - var gotChunks []*TOCEntry - var last *TOCEntry - for off := int64(0); off < f.Size; off++ { - e, ok := r.ChunkEntryForOffset(file, off) - if !ok { - t.Errorf("no ChunkEntryForOffset at %d", off) - return - } - if last != e { - gotChunks = append(gotChunks, e) - last = e - } - } - if !reflect.DeepEqual(chunks, gotChunks) { - t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks) - } - - // And verify the NextOffset - for i := 0; i < len(gotChunks)-1; i++ { - ci := gotChunks[i] - cnext := gotChunks[i+1] - if ci.NextOffset() != cnext.Offset { - t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset) - } - } - }) -} - -func entryHasChildren(dir string, want ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - want := append([]string(nil), want...) - var got []string - ent, ok := r.Lookup(dir) - if !ok { - t.Fatalf("didn't find TOCEntry for dir node %q", dir) - } - for baseName := range ent.children { - got = append(got, baseName) - } - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { - t.Errorf("children of %q = %q; want %q", dir, got, want) - } - }) -} - -func hasDir(file string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasDirLinkCount(file string, count int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - return - } - if ent.NumLink != count { - t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasMode(file string, mode os.FileMode) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Stat().Mode() != mode { - t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode) - return - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasSymlink(file, target string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "symlink" { - t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type) - } else if ent.LinkName != target { - t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target) - } - return - } - } - t.Errorf("symlink %q not found", file) - }) -} - -func lookupMatch(name string, want *TOCEntry) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup(name) - if !ok { - t.Fatalf("failed to Lookup entry %q", name) - } - if !reflect.DeepEqual(e, want) { - t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want) - } - - }) -} - -func hasEntryOwner(entry string, owner owner) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(strings.TrimSuffix(entry, "/")) - if !ok { - t.Errorf("entry %q not found", entry) - return - } - if ent.UID != owner.uid || ent.GID != owner.gid { - t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid) - return - } - }) -} - -func mustSameEntry(files ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - var first *TOCEntry - for _, f := range files { - if first == nil { - var ok bool - first, ok = r.Lookup(f) - if !ok { - t.Errorf("unknown first file on Lookup: %q", f) - return - } - } - - // Test Lookup - e, ok := r.Lookup(f) - if !ok { - t.Errorf("unknown file on Lookup: %q", f) - return - } - if e != first { - t.Errorf("Lookup: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test LookupChild - pe, ok := r.Lookup(filepath.Dir(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get parent of %q", f) - return - } - e, ok = pe.LookupChild(filepath.Base(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get %q as the child of %+v", f, pe) - return - } - if e != first { - t.Errorf("LookupChild: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test ForeachChild - pe.ForeachChild(func(baseName string, e *TOCEntry) bool { - if baseName == filepath.Base(filepath.Clean(f)) { - if e != first { - t.Errorf("ForeachChild: %+v(%p) != %+v(%p)", e, e, first, first) - return false - } - } - return true - }) - } - }) -} - -func viewContent(c []byte) string { - if len(c) < 100 { - return string(c) - } - return string(c[:50]) + "...(omit)..." + string(c[50:100]) -} - -func tarOf(s ...tarEntry) []tarEntry { return s } - -type tarEntry interface { - appendTar(tw *tar.Writer, prefix string, format tar.Format) error -} - -type tarEntryFunc func(*tar.Writer, string, tar.Format) error - -func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format) error { - return f(tw, prefix, format) -} - -func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { - format := tar.FormatUnknown - for _, opt := range opts { - switch v := opt.(type) { - case tar.Format: - format = v - default: - panic(fmt.Errorf("unsupported opt for buildTar: %v", opt)) - } - } - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix, format); err != nil { - t.Fatalf("building input tar: %v", err) - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - data := append(buf.Bytes(), make([]byte, 100)...) // append empty bytes at the tail to see lossless works - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) -} - -func dir(name string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var o owner - mode := os.FileMode(0755) - for _, opt := range opts { - switch v := opt.(type) { - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if !strings.HasSuffix(name, "/") { - panic(fmt.Sprintf("missing trailing slash in dir %q ", name)) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeDir, - Name: prefix + name, - Mode: tm, - Uid: o.uid, - Gid: o.gid, - Format: format, - }) - }) -} - -// xAttr are extended attributes to set on test files created with the file func. -type xAttr map[string]string - -// owner is owner ot set on test files and directories with the file and dir functions. -type owner struct { - uid int - gid int -} - -func file(name, contents string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var xattrs xAttr - var o owner - mode := os.FileMode(0644) - for _, opt := range opts { - switch v := opt.(type) { - case xAttr: - xattrs = v - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if strings.HasSuffix(name, "/") { - return fmt.Errorf("bogus trailing slash in file %q", name) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - if len(xattrs) > 0 { - format = tar.FormatPAX // only PAX supports xattrs - } - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Mode: tm, - Xattrs: xattrs, - Size: int64(len(contents)), - Uid: o.uid, - Gid: o.gid, - Format: format, - }); err != nil { - return err - } - _, err = io.WriteString(tw, contents) - return err - }) -} - -func symlink(name, target string) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeSymlink, - Name: prefix + name, - Linkname: target, - Mode: 0644, - Format: format, - }) - }) -} - -func link(name string, linkname string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeLink, - Name: prefix + name, - Linkname: linkname, - ModTime: now, - Format: format, - }) - }) -} - -func chardev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeChar, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} - -func blockdev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeBlock, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} -func fifo(name string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeFifo, - Name: prefix + name, - ModTime: now, - Format: format, - }) - }) -} - -func prefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func noPrefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func regDigest(t TestingT, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry { - if digestMap == nil { - t.Fatalf("digest map mustn't be nil") - } - content := []byte(contentStr) - - var n int64 - for n < int64(len(content)) { - size := int64(chunkSize) - remain := int64(len(content)) - n - if remain < size { - size = remain - } - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil { - t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)", - string(content[n:n+size]), name, n, size) - } - digestMap[chunkID(name, n, size)] = dgstr.Digest() - n += size - } - - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Size: int64(len(content)), - Format: format, - }); err != nil { - return err - } - if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil { - return err - } - return nil - }) -} - -var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randomContents(n int) string { - b := make([]rune, n) - for i := range b { - bi, err := rand.Int(rand.Reader, big.NewInt(int64(len(runes)))) - if err != nil { - panic(err) - } - b[i] = runes[int(bi.Int64())] - } - return string(b) -} - -func fileModeToTarMode(mode os.FileMode) (int64, error) { - h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "") - if err != nil { - return 0, err - } - return h.Mode, nil -} - -// fileInfoOnlyMode is os.FileMode that populates only file mode. -type fileInfoOnlyMode os.FileMode - -func (f fileInfoOnlyMode) Name() string { return "" } -func (f fileInfoOnlyMode) Size() int64 { return 0 } -func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } -func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } -func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } - -func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) { - if len(streams) == 0 { - return // nop - } - - wants := map[int64]struct{}{} - for _, s := range streams { - wants[s] = struct{}{} - } - - len0 := len(b) - br := bytes.NewReader(b) - zr := new(gzip.Reader) - t.Logf("got gzip streams:") - numStreams := 0 - for { - zoff := len0 - br.Len() - if err := zr.Reset(br); err != nil { - if err == io.EOF { - return - } - t.Fatalf("countStreams(gzip), Reset: %v", err) - } - zr.Multistream(false) - n, err := io.Copy(io.Discard, zr) - if err != nil { - t.Fatalf("countStreams(gzip), Copy: %v", err) - } - var extra string - if len(zr.Extra) > 0 { - extra = fmt.Sprintf("; extra=%q", zr.Extra) - } - t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra) - delete(wants, int64(zoff)) - numStreams++ - } -} - -func GzipDiffIDOf(t TestingT, b []byte) string { - h := sha256.New() - zr, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("diffIDOf(gzip): %v", err) - } - defer zr.Close() - if _, err := io.Copy(h, zr); err != nil { - t.Fatalf("diffIDOf(gzip).Copy: %v", err) - } - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go deleted file mode 100644 index 57e0aa614..000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ /dev/null @@ -1,342 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "hash" - "io" - "os" - "path" - "time" - - digest "github.com/opencontainers/go-digest" -) - -const ( - // TOCTarName is the name of the JSON file in the tar archive in the - // table of contents gzip stream. - TOCTarName = "stargz.index.json" - - // FooterSize is the number of bytes in the footer - // - // The footer is an empty gzip stream with no compression and an Extra - // header of the form "%016xSTARGZ", where the 64 bit hex-encoded - // number is the offset to the gzip stream of JSON TOC. - // - // 51 comes from: - // - // 10 bytes gzip header - // 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ")) - // 2 bytes Extra: SI1 = 'S', SI2 = 'G' - // 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ")) - // 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC) - // 5 bytes flate header - // 8 bytes gzip footer - // (End of the eStargz blob) - // - // NOTE: For Extra fields, subfield IDs SI1='S' SI2='G' is used for eStargz. - FooterSize = 51 - - // legacyFooterSize is the number of bytes in the legacy stargz footer. - // - // 47 comes from: - // - // 10 byte gzip header + - // 2 byte (LE16) length of extra, encoding 22 (16 hex digits + len("STARGZ")) == "\x16\x00" + - // 22 bytes of extra (fmt.Sprintf("%016xSTARGZ", tocGzipOffset)) - // 5 byte flate header - // 8 byte gzip footer (two little endian uint32s: digest, size) - legacyFooterSize = 47 - - // TOCJSONDigestAnnotation is an annotation for an image layer. This stores the - // digest of the TOC JSON. - // This annotation is valid only when it is specified in `.[]layers.annotations` - // of an image manifest. - TOCJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest" - - // StoreUncompressedSizeAnnotation is an additional annotation key for eStargz to enable lazy - // pulling on containers/storage. Stargz Store is required to expose the layer's uncompressed size - // to the runtime but current OCI image doesn't ship this information by default. So we store this - // to the special annotation. - StoreUncompressedSizeAnnotation = "io.containers.estargz.uncompressed-size" - - // PrefetchLandmark is a file entry which indicates the end position of - // prefetch in the stargz file. - PrefetchLandmark = ".prefetch.landmark" - - // NoPrefetchLandmark is a file entry which indicates that no prefetch should - // occur in the stargz file. - NoPrefetchLandmark = ".no.prefetch.landmark" - - landmarkContents = 0xf -) - -// JTOC is the JSON-serialized table of contents index of the files in the stargz file. -type JTOC struct { - Version int `json:"version"` - Entries []*TOCEntry `json:"entries"` -} - -// TOCEntry is an entry in the stargz file's TOC (Table of Contents). -type TOCEntry struct { - // Name is the tar entry's name. It is the complete path - // stored in the tar file, not just the base name. - Name string `json:"name"` - - // Type is one of "dir", "reg", "symlink", "hardlink", "char", - // "block", "fifo", or "chunk". - // The "chunk" type is used for regular file data chunks past the first - // TOCEntry; the 2nd chunk and on have only Type ("chunk"), Offset, - // ChunkOffset, and ChunkSize populated. - Type string `json:"type"` - - // Size, for regular files, is the logical size of the file. - Size int64 `json:"size,omitempty"` - - // ModTime3339 is the modification time of the tar entry. Empty - // means zero or unknown. Otherwise it's in UTC RFC3339 - // format. Use the ModTime method to access the time.Time value. - ModTime3339 string `json:"modtime,omitempty"` - modTime time.Time - - // LinkName, for symlinks and hardlinks, is the link target. - LinkName string `json:"linkName,omitempty"` - - // Mode is the permission and mode bits. - Mode int64 `json:"mode,omitempty"` - - // UID is the user ID of the owner. - UID int `json:"uid,omitempty"` - - // GID is the group ID of the owner. - GID int `json:"gid,omitempty"` - - // Uname is the username of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same UID. - Uname string `json:"userName,omitempty"` - - // Gname is the group name of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same GID. - Gname string `json:"groupName,omitempty"` - - // Offset, for regular files, provides the offset in the - // stargz file to the file's data bytes. See ChunkOffset and - // ChunkSize. - Offset int64 `json:"offset,omitempty"` - - // InnerOffset is an optional field indicates uncompressed offset - // of this "reg" or "chunk" payload in a stream starts from Offset. - // This field enables to put multiple "reg" or "chunk" payloads - // in one chunk with having the same Offset but different InnerOffset. - InnerOffset int64 `json:"innerOffset,omitempty"` - - nextOffset int64 // the Offset of the next entry with a non-zero Offset - - // DevMajor is the major device number for "char" and "block" types. - DevMajor int `json:"devMajor,omitempty"` - - // DevMinor is the major device number for "char" and "block" types. - DevMinor int `json:"devMinor,omitempty"` - - // NumLink is the number of entry names pointing to this entry. - // Zero means one name references this entry. - // This field is calculated during runtime and not recorded in TOC JSON. - NumLink int `json:"-"` - - // Xattrs are the extended attribute for the entry. - Xattrs map[string][]byte `json:"xattrs,omitempty"` - - // Digest stores the OCI checksum for regular files payload. - // It has the form "sha256:abcdef01234....". - Digest string `json:"digest,omitempty"` - - // ChunkOffset is non-zero if this is a chunk of a large, - // regular file. If so, the Offset is where the gzip header of - // ChunkSize bytes at ChunkOffset in Name begin. - // - // In serialized form, a "chunkSize" JSON field of zero means - // that the chunk goes to the end of the file. After reading - // from the stargz TOC, though, the ChunkSize is initialized - // to a non-zero file for when Type is either "reg" or - // "chunk". - ChunkOffset int64 `json:"chunkOffset,omitempty"` - ChunkSize int64 `json:"chunkSize,omitempty"` - - // ChunkDigest stores an OCI digest of the chunk. This must be formed - // as "sha256:0123abcd...". - ChunkDigest string `json:"chunkDigest,omitempty"` - - children map[string]*TOCEntry - - // chunkTopIndex is index of the entry where Offset starts in the blob. - chunkTopIndex int -} - -// ModTime returns the entry's modification time. -func (e *TOCEntry) ModTime() time.Time { return e.modTime } - -// NextOffset returns the position (relative to the start of the -// stargz file) of the next gzip boundary after e.Offset. -func (e *TOCEntry) NextOffset() int64 { return e.nextOffset } - -func (e *TOCEntry) addChild(baseName string, child *TOCEntry) { - if e.children == nil { - e.children = make(map[string]*TOCEntry) - } - if child.Type == "dir" { - e.NumLink++ // Entry ".." in the subdirectory links to this directory - } - e.children[baseName] = child -} - -// isDataType reports whether TOCEntry is a regular file or chunk (something that -// contains regular file data). -func (e *TOCEntry) isDataType() bool { return e.Type == "reg" || e.Type == "chunk" } - -// Stat returns a FileInfo value representing e. -func (e *TOCEntry) Stat() os.FileInfo { return fileInfo{e} } - -// ForeachChild calls f for each child item. If f returns false, iteration ends. -// If e is not a directory, f is not called. -func (e *TOCEntry) ForeachChild(f func(baseName string, ent *TOCEntry) bool) { - for name, ent := range e.children { - if !f(name, ent) { - return - } - } -} - -// LookupChild returns the directory e's child by its base name. -func (e *TOCEntry) LookupChild(baseName string) (child *TOCEntry, ok bool) { - child, ok = e.children[baseName] - return -} - -// fileInfo implements os.FileInfo using the wrapped *TOCEntry. -type fileInfo struct{ e *TOCEntry } - -var _ os.FileInfo = fileInfo{} - -func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } -func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } -func (fi fileInfo) Size() int64 { return fi.e.Size } -func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() } -func (fi fileInfo) Sys() interface{} { return fi.e } -func (fi fileInfo) Mode() (m os.FileMode) { - // TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg. - m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() & - (os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky) - switch fi.e.Type { - case "dir": - m |= os.ModeDir - case "symlink": - m |= os.ModeSymlink - case "char": - m |= os.ModeDevice | os.ModeCharDevice - case "block": - m |= os.ModeDevice - case "fifo": - m |= os.ModeNamedPipe - } - return m -} - -// TOCEntryVerifier holds verifiers that are usable for verifying chunks contained -// in a eStargz blob. -type TOCEntryVerifier interface { - - // Verifier provides a content verifier that can be used for verifying the - // contents of the specified TOCEntry. - Verifier(ce *TOCEntry) (digest.Verifier, error) -} - -// Compression provides the compression helper to be used creating and parsing eStargz. -// This package provides gzip-based Compression by default, but any compression -// algorithm (e.g. zstd) can be used as long as it implements Compression. -type Compression interface { - Compressor - Decompressor -} - -// Compressor represents the helper mothods to be used for creating eStargz. -type Compressor interface { - // Writer returns WriteCloser to be used for writing a chunk to eStargz. - // Everytime a chunk is written, the WriteCloser is closed and Writer is - // called again for writing the next chunk. - // - // The returned writer should implement "Flush() error" function that flushes - // any pending compressed data to the underlying writer. - Writer(w io.Writer) (WriteFlushCloser, error) - - // WriteTOCAndFooter is called to write JTOC to the passed Writer. - // diffHash calculates the DiffID (uncompressed sha256 hash) of the blob - // WriteTOCAndFooter can optionally write anything that affects DiffID calculation - // (e.g. uncompressed TOC JSON). - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob when it's parsed. - WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error) -} - -// Decompressor represents the helper mothods to be used for parsing eStargz. -type Decompressor interface { - // Reader returns ReadCloser to be used for decompressing file payload. - Reader(r io.Reader) (io.ReadCloser, error) - - // FooterSize returns the size of the footer of this blob. - FooterSize() int64 - - // ParseFooter parses the footer and returns the offset and (compressed) size of TOC. - // payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between - // the top until the TOC JSON). - // - // If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader - // to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it. - // - // tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the - // footer (blob size - tocOff - FooterSize). - // If blobPayloadSize < 0, blobPayloadSize become the blob size. - ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) - - // ParseTOC parses TOC from the passed reader. The reader provides the partial contents - // of the underlying blob that has the range specified by ParseFooter method. - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob. This must match to the value returned from - // Compressor.WriteTOCAndFooter that is used when creating this blob. - // - // If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob. - // Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location - // and return it. - ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) -} - -type WriteFlushCloser interface { - io.WriteCloser - Flush() error -} diff --git a/vendor/github.com/docker/cli/AUTHORS b/vendor/github.com/docker/cli/AUTHORS index accbf6c52..c42a3f8f4 100644 --- a/vendor/github.com/docker/cli/AUTHORS +++ b/vendor/github.com/docker/cli/AUTHORS @@ -43,6 +43,7 @@ Alexander Larsson Alexander Morozov Alexander Ryabov Alexandre González +Alexandre Levavasseur Alexandre Vallières-Lagacé Alexey Igrychev Alexis Couvreur @@ -161,6 +162,7 @@ Chen Chuanliang Chen Hanxiao Chen Mingjie Chen Qiu +Ching Wei Kang <164879897+WilliamK112@users.noreply.github.com> Chris Chinchilla Chris Couzens Chris Gavin @@ -366,7 +368,7 @@ Hugo Gabriel Eyherabide huqun Huu Nguyen Hyzhou Zhy -Iain MacDonald +Iain MacDonald Iain Samuel McLean Elder Ian Campbell Ian Philpot @@ -552,6 +554,7 @@ Ludovic Temgoua Abanda Luis Henrique Mulinari Luka Hartwig Lukas Heeren +Lukas Michael Lukasz Zajaczkowski Luo Jiyin Lydell Manganti @@ -562,6 +565,7 @@ Maciej Kalisz Madhav Puri Madhu Venugopal Madhur Batra +Mahesh Thakur Malte Janduda Manjunath A Kumatagi Mansi Nahar @@ -589,10 +593,12 @@ Mathieu Rollet Matt Gucci Matt Robenolt Matteo Orefice +Matteo Panzeri Matthew Heon Matthieu Hauglustaine Matthieu MOREL Mauro Porras P +Max Morozov Max Shytikov Max-Julian Pogner Maxime Petazzoni @@ -617,6 +623,7 @@ Michael West Michael Zampani Michal Minář Michał Czeraszkiewicz +Mickael Emirkanian Miguel Angel Alvarez Cabrerizo Mihai Borobocea Mihuleacc Sergiu @@ -638,6 +645,7 @@ Mohammad Banikazemi Mohammad Hossein Mohammed Aaqib Ansari Mohammed Aminu Futa +Mohammed Thaha Mohini Anne Dsouza Moorthy RS Morgan Bauer @@ -689,6 +697,7 @@ Olli Janatuinen Oscar Wieman Otto Kekäläinen Ovidio Mallo +Park Jaeon Pascal Borreli Patrick Böänziger Patrick Daigle <114765035+pdaig@users.noreply.github.com> @@ -715,6 +724,7 @@ Peter Jaffe Peter Kehl Peter Nagy Peter Salvatore +Peter Valdemar Mørch Peter Waller Phil Estes Philip Alexander Etling @@ -739,6 +749,7 @@ Ray Tsang Reficul Remy Suen Renaud Gaubert +René Hermenau Ricardo N Feliciano Rich Moyse Richard Chen Zheng <58443436+rchenzheng@users.noreply.github.com> @@ -789,6 +800,7 @@ Scott Collier Sean Christopherson Sean Rodman Sebastiaan van Stijn +Seiya Miyata Sergey Tryuber Serhat Gülçiçek Sevki Hasirci @@ -889,8 +901,10 @@ Umesh Yadav Vaclav Struhar Valentin Lorentz Vardan Pogosian +Varun Hotani Venkateswara Reddy Bukkasamudram Veres Lajos +Vibhu Anan Victor Vieux Victoria Bialas Viktor Stanchev diff --git a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go index e3ef8e25e..d4037b7a4 100644 --- a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go +++ b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go @@ -104,7 +104,7 @@ func (c *fileStore) Store(authConfig types.AuthConfig) error { // stored as hostname or as hostname including scheme (in legacy configuration // files). // -// It's the equivalent to [registry.ConvertToHostname] in the daemon. +// It's based on [registry.ConvertToHostname] from Moby daemon. // // [registry.ConvertToHostname]: https://pkg.go.dev/github.com/moby/moby/v2@v2.0.0-beta.7/daemon/pkg/registry#ConvertToHostname func ConvertToHostname(maybeURL string) string { @@ -117,7 +117,48 @@ func ConvertToHostname(maybeURL string) string { } return net.JoinHostPort(u.Hostname(), u.Port()) } + + if hostName := hostFromURLFallback(stripped); hostName != "" { + return hostName + } } hostName, _, _ := strings.Cut(stripped, "/") return hostName } + +// hostFromURLFallback extracts a host from scheme URLs that net/url rejects. +// Go rejects unbracketed IPv6 literals in URL hosts since +// https://github.com/golang/go/commit/0c28789bd7dfc55099cac86a3212dda0d6c091f6 +func hostFromURLFallback(maybeURL string) string { + _, rest, ok := strings.Cut(maybeURL, "://") + if !ok { + return "" + } + + hostName, _, _ := strings.Cut(rest, "/") + if hostName == "" { + return "" + } + + if strings.Count(hostName, ":") > 1 && !strings.HasPrefix(hostName, "[") { + portStart := strings.LastIndex(hostName, ":") + addr, port := hostName[:portStart], hostName[portStart+1:] + if addr != "" && isPort(port) { + return net.JoinHostPort(addr, port) + } + } + + return hostName +} + +func isPort(port string) bool { + if port == "" { + return false + } + for _, r := range port { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/vendor/github.com/go-logr/logr/context_noslog.go b/vendor/github.com/go-logr/logr/context_noslog.go index f012f9a18..0a3d1a125 100644 --- a/vendor/github.com/go-logr/logr/context_noslog.go +++ b/vendor/github.com/go-logr/logr/context_noslog.go @@ -1,5 +1,4 @@ //go:build !go1.21 -// +build !go1.21 /* Copyright 2019 The logr Authors. diff --git a/vendor/github.com/go-logr/logr/context_slog.go b/vendor/github.com/go-logr/logr/context_slog.go index 065ef0b82..c69eb01ba 100644 --- a/vendor/github.com/go-logr/logr/context_slog.go +++ b/vendor/github.com/go-logr/logr/context_slog.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2019 The logr Authors. diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go index b22c57d71..7f4996e9e 100644 --- a/vendor/github.com/go-logr/logr/funcr/funcr.go +++ b/vendor/github.com/go-logr/logr/funcr/funcr.go @@ -426,7 +426,7 @@ func (f Formatter) colon() byte { } func (f Formatter) pretty(value any) string { - return f.prettyWithFlags(value, 0, 0) + return f.prettyWithFlags(value, 0, 0, 0, nil) } const ( @@ -434,7 +434,13 @@ const ( ) // TODO: This is not fast. Most of the overhead goes here. -func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { +// value: The value to render +// flags: Bitmask of flags (see above) +// depth: The current depth of nested structs, slices, arrays, and maps +// ptrDepth: The current depth of including pointer dereferences +// ptrMap: A map of pointers already seen, to avoid infinite recursion (usually +// nil unless ptrDepth is large) +func (f Formatter) prettyWithFlags(value any, flags uint32, depth int, ptrDepth int, ptrMap map[uintptr]bool) string { if depth > f.opts.MaxLogDepth { return `""` } @@ -504,7 +510,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { // arbitrary keys might need escaping buf.WriteString(prettyString(k)) buf.WriteByte(f.colon()) - buf.WriteString(f.prettyWithFlags(v[i+1], 0, depth+1)) + buf.WriteString(f.prettyWithFlags(v[i+1], 0, depth+1, ptrDepth+1, ptrMap)) } if flags&flagRawStruct == 0 { buf.WriteByte('}') @@ -576,7 +582,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { } printComma = true // if we got here, we are rendering a field if fld.Anonymous && fld.Type.Kind() == reflect.Struct && name == "" { - buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), flags|flagRawStruct, depth+1)) + buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), flags|flagRawStruct, depth+1, ptrDepth+1, ptrMap)) continue } if name == "" { @@ -585,7 +591,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { // field names can't contain characters which need escaping buf.WriteString(f.quoted(name, false)) buf.WriteByte(f.colon()) - buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), 0, depth+1)) + buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), 0, depth+1, ptrDepth+1, ptrMap)) } if flags&flagRawStruct == 0 { buf.WriteByte('}') @@ -612,7 +618,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { buf.WriteByte(f.comma()) } e := v.Index(i) - buf.WriteString(f.prettyWithFlags(e.Interface(), 0, depth+1)) + buf.WriteString(f.prettyWithFlags(e.Interface(), 0, depth+1, ptrDepth+1, ptrMap)) } buf.WriteByte(']') return buf.String() @@ -637,7 +643,8 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { keystr = prettyString(keystr) } else { // prettyWithFlags will produce already-escaped values - keystr = f.prettyWithFlags(it.Key().Interface(), 0, depth+1) + // key depth is unrelated to overall depth + keystr = f.prettyWithFlags(it.Key().Interface(), 0, 0, ptrDepth, ptrMap) if t.Key().Kind() != reflect.String { // JSON only does string keys. Unlike Go's standard JSON, we'll // convert just about anything to a string. @@ -646,16 +653,34 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string { } buf.WriteString(keystr) buf.WriteByte(f.colon()) - buf.WriteString(f.prettyWithFlags(it.Value().Interface(), 0, depth+1)) + buf.WriteString(f.prettyWithFlags(it.Value().Interface(), 0, depth+1, ptrDepth+1, ptrMap)) i++ } buf.WriteByte('}') return buf.String() - case reflect.Ptr, reflect.Interface: + case reflect.Pointer, reflect.Interface: if v.IsNil() { return "null" } - return f.prettyWithFlags(v.Elem().Interface(), 0, depth) + // Special case: recursive pointers. For normal use we do not want to + // count pointer dereferences as depth, but if we see the same pointer + // again we have a recursion and need to stop. After a large number of + // pointer dereferences we will start tracking pointers to avoid the + // perf hit of doing it in the normal path. + // + // This should not happen accidentally (e.g. json decoding should never + // do this) but we can handle it gracefully. + if ptrMap != nil && ptrMap[uintptr(v.Pointer())] { + depth = f.opts.MaxLogDepth + 1 // force a depth error + } + const maxDepthFactor = 4 // arbitrary, but we want it large enough to not false-alert + if ptrDepth > f.opts.MaxLogDepth*maxDepthFactor && ptrMap == nil { + ptrMap = map[uintptr]bool{} + } + if ptrMap != nil { + ptrMap[(uintptr)(v.Pointer())] = true + } + return f.prettyWithFlags(v.Elem().Interface(), 0, depth, ptrDepth+1, ptrMap) } return fmt.Sprintf(`""`, t.Kind().String()) } @@ -697,7 +722,7 @@ func isEmpty(v reflect.Value) bool { return v.Float() == 0 case reflect.Complex64, reflect.Complex128: return v.Complex() == 0 - case reflect.Interface, reflect.Ptr: + case reflect.Interface, reflect.Pointer: return v.IsNil() } return false diff --git a/vendor/github.com/go-logr/logr/funcr/slogsink.go b/vendor/github.com/go-logr/logr/funcr/slogsink.go index 7bd84761e..8b519c91e 100644 --- a/vendor/github.com/go-logr/logr/funcr/slogsink.go +++ b/vendor/github.com/go-logr/logr/funcr/slogsink.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2023 The logr Authors. @@ -33,7 +32,7 @@ const extraSlogSinkDepth = 3 // 2 for slog, 1 for SlogSink func (l fnlogger) Handle(_ context.Context, record slog.Record) error { kvList := make([]any, 0, 2*record.NumAttrs()) record.Attrs(func(attr slog.Attr) bool { - kvList = attrToKVs(attr, kvList) + kvList = attrToKVs(attr, kvList, l.opts.MaxLogDepth) return true }) @@ -49,7 +48,7 @@ func (l fnlogger) Handle(_ context.Context, record slog.Record) error { func (l fnlogger) WithAttrs(attrs []slog.Attr) logr.SlogSink { kvList := make([]any, 0, 2*len(attrs)) for _, attr := range attrs { - kvList = attrToKVs(attr, kvList) + kvList = attrToKVs(attr, kvList, l.opts.MaxLogDepth) } l.AddValues(kvList) return &l @@ -61,14 +60,25 @@ func (l fnlogger) WithGroup(name string) logr.SlogSink { } // attrToKVs appends a slog.Attr to a logr-style kvList. It handle slog Groups -// and other details of slog. -func attrToKVs(attr slog.Attr, kvList []any) []any { +// and other details of slog. maxDepth bounds recursion into nested groups so a +// deeply-nested slog.Group cannot exhaust the stack; it is decremented per group +// level and starts at the Formatter's MaxLogDepth (past which the formatter would +// truncate the rendering anyway). +func attrToKVs(attr slog.Attr, kvList []any, maxDepth int) []any { attrVal := attr.Value.Resolve() if attrVal.Kind() == slog.KindGroup { + if maxDepth <= 0 { + // Nesting is too deep to build without risking a stack overflow. + // Stop here; the formatter truncates below MaxLogDepth regardless. + if attr.Key != "" { + kvList = append(kvList, attr.Key, "") + } + return kvList + } groupVal := attrVal.Group() grpKVs := make([]any, 0, 2*len(groupVal)) for _, attr := range groupVal { - grpKVs = attrToKVs(attr, grpKVs) + grpKVs = attrToKVs(attr, grpKVs, maxDepth-1) } if attr.Key == "" { // slog says we have to inline these diff --git a/vendor/github.com/go-logr/logr/sloghandler.go b/vendor/github.com/go-logr/logr/sloghandler.go index 82d1ba494..befaf5510 100644 --- a/vendor/github.com/go-logr/logr/sloghandler.go +++ b/vendor/github.com/go-logr/logr/sloghandler.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2023 The logr Authors. diff --git a/vendor/github.com/go-logr/logr/slogr.go b/vendor/github.com/go-logr/logr/slogr.go index 28a83d024..bfe80eb8d 100644 --- a/vendor/github.com/go-logr/logr/slogr.go +++ b/vendor/github.com/go-logr/logr/slogr.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2023 The logr Authors. diff --git a/vendor/github.com/go-logr/logr/slogr/slogr.go b/vendor/github.com/go-logr/logr/slogr/slogr.go index 36432c56f..933e715b2 100644 --- a/vendor/github.com/go-logr/logr/slogr/slogr.go +++ b/vendor/github.com/go-logr/logr/slogr/slogr.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2023 The logr Authors. diff --git a/vendor/github.com/go-logr/logr/slogsink.go b/vendor/github.com/go-logr/logr/slogsink.go index 4060fcbc2..ab76ea99f 100644 --- a/vendor/github.com/go-logr/logr/slogsink.go +++ b/vendor/github.com/go-logr/logr/slogsink.go @@ -1,5 +1,4 @@ //go:build go1.21 -// +build go1.21 /* Copyright 2023 The logr Authors. diff --git a/vendor/github.com/google/cel-go/cel/BUILD.bazel b/vendor/github.com/google/cel-go/cel/BUILD.bazel index 46cb26d62..62a56036a 100644 --- a/vendor/github.com/google/cel-go/cel/BUILD.bazel +++ b/vendor/github.com/google/cel-go/cel/BUILD.bazel @@ -26,6 +26,7 @@ go_library( importpath = "github.com/google/cel-go/cel", visibility = ["//visibility:public"], deps = [ + "//cel/async:go_default_library", "//checker:go_default_library", "//checker/decls:go_default_library", "//common:go_default_library", @@ -70,6 +71,7 @@ go_test( "inlining_test.go", "io_test.go", "optimizer_test.go", + "program_async_test.go", "prompt_test.go", "validator_test.go", ], @@ -84,6 +86,7 @@ go_test( "//cel/testdata:test_fds_with_source_info", ], deps = [ + "//cel/async:go_default_library", "//common/operators:go_default_library", "//common/overloads:go_default_library", "//common/types:go_default_library", diff --git a/vendor/github.com/google/cel-go/cel/async/BUILD.bazel b/vendor/github.com/google/cel-go/cel/async/BUILD.bazel new file mode 100644 index 000000000..85b28bcdb --- /dev/null +++ b/vendor/github.com/google/cel-go/cel/async/BUILD.bazel @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +package( + licenses = ["notice"], # Apache 2.0 +) + +go_library( + name = "go_default_library", + srcs = [ + "async.go", + ], + importpath = "github.com/google/cel-go/cel/async", + visibility = ["//visibility:public"], + deps = [ + "//common/decls:go_default_library", + "//common/functions:go_default_library", + "//common/types:go_default_library", + "//common/types/ref:go_default_library", + "//interpreter:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "async_test.go", + ], + deps = [ + ":go_default_library", + "//common/decls:go_default_library", + "//common/functions:go_default_library", + "//common/types:go_default_library", + "//common/types/ref:go_default_library", + ], +) diff --git a/vendor/github.com/google/cel-go/cel/async/async.go b/vendor/github.com/google/cel-go/cel/async/async.go new file mode 100644 index 000000000..a011114bd --- /dev/null +++ b/vendor/github.com/google/cel-go/cel/async/async.go @@ -0,0 +1,235 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package async provides helpers for configuring and executing asynchronous CEL functions, +// including drain strategies, retry, timeout, concurrency limiting, and caching wrappers. +package async + +import ( + "context" + "errors" + "time" + + "github.com/google/cel-go/common/decls" + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/interpreter" +) + +// Call describes a pending or completed asynchronous function call. +// This interface exposes a safe, read-only view of the internal interpreter state. +type Call = interpreter.AsyncCall + +// Observer provides callbacks for monitoring the lifecycle of asynchronous function calls. +// +// Implementations must be safe for concurrent use: the start and finish callbacks run on different +// goroutines, and finish callbacks for distinct calls may run concurrently. See +// interpreter.AsyncObserver for details. +type Observer = interpreter.AsyncObserver + +// BlockingOp is a blocking asynchronous function operation. +type BlockingOp = functions.BlockingAsyncOp + +// DrainAction dictates what ConcurrentEval should do after inspecting completions. +type DrainAction struct { + // Reevaluate indicates that the AST should be re-evaluated immediately. + // If true, WaitDuration is ignored. + Reevaluate bool + // WaitDuration indicates how long the evaluator should wait for additional + // completions before deciding to re-evaluate. A duration of 0 means wait + // indefinitely (block on the next completion). + WaitDuration time.Duration +} + +// DrainStrategy controls when ConcurrentEval re-evaluates after async completions. +// +// The evaluator consults the strategy each time a completion is received. +type DrainStrategy interface { + // NextAction evaluates the current state of asynchronous evaluation and + // determines the next step. + // + // - completed: The set of completions accumulated in the current batch. + // - active: The number of async calls currently launched but unresolved. + NextAction(completed []Call, active int) DrainAction +} + +// DrainNone returns a strategy that re-evaluates after every single completion. +// This is the default strategy. +func DrainNone() DrainStrategy { + return drainNone{} +} + +type drainNone struct{} + +func (drainNone) NextAction(completed []Call, active int) DrainAction { + return DrainAction{Reevaluate: active == 0 || len(completed) > 0} +} + +// DrainReady returns a strategy that waits for a short duration after the first +// completion to batch any other functions that complete at roughly the same time. +func DrainReady(debounce time.Duration) DrainStrategy { + return drainReady{debounce: debounce} +} + +type drainReady struct { + debounce time.Duration +} + +func (d drainReady) NextAction(completed []Call, active int) DrainAction { + if active == 0 { + return DrainAction{Reevaluate: true} // Nothing left to wait for + } + if len(completed) == 0 { + return DrainAction{Reevaluate: false, WaitDuration: 0} // Wait indefinitely for first + } + return DrainAction{Reevaluate: false, WaitDuration: d.debounce} // Wait for debounce period +} + +// DrainAll returns a strategy that waits for all currently pending calls to +// complete before re-evaluating. +// +// Note: This strategy is optimal for independent async calls, but will over-wait +// if some calls depend on the results of others. +func DrainAll() DrainStrategy { + return drainAll{} +} + +type drainAll struct{} + +func (drainAll) NextAction(completed []Call, active int) DrainAction { + return DrainAction{Reevaluate: active == 0} +} + +// Timeout wraps a BlockingAsyncOp with a per-call timeout. +// +// The timeout is enforced even when the wrapped function ignores its context: the function runs on +// its own goroutine and Timeout selects on the deadline, returning a timeout error when it +// fires. A function that ignores cancellation cannot be forcibly stopped (Go cannot kill a +// goroutine), so its goroutine continues running in the background until it returns on its own; +// only its result is abandoned. This is the recommended way to bound functions that may hang or +// are not under the caller's control. The extra goroutine is incurred only by Timeout-wrapped +// calls, not by async evaluation in general. +func Timeout(fn functions.BlockingAsyncOp, timeout time.Duration) functions.BlockingAsyncOp { + return func(ctx context.Context, args ...ref.Val) ref.Val { + tCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resCh := make(chan ref.Val, 1) + go func() { resCh <- fn(tCtx, args...) }() + select { + case res := <-resCh: + return res + case <-tCtx.Done(): + return types.NewErr("operation timed out after %v: %v", timeout, tCtx.Err()) + } + } +} + +// TimeoutBinding wraps a BlockingAsyncOp with a per-call timeout and returns an OverloadOpt. +func TimeoutBinding(fn functions.BlockingAsyncOp, timeout time.Duration) decls.OverloadOpt { + return decls.AsyncBinding(Timeout(fn, timeout)) +} + +// RetryOption configures the behavior of RetryBinding. +type RetryOption func(*retryConfig) + +type retryConfig struct { + maxAttempts int + backoff time.Duration +} + +// RetryAttempts sets the maximum number of attempts (including the first one). +func RetryAttempts(attempts int) RetryOption { + return func(c *retryConfig) { + c.maxAttempts = attempts + } +} + +// RetryBackoff sets the fixed backoff duration between attempts. +func RetryBackoff(backoff time.Duration) RetryOption { + return func(c *retryConfig) { + c.backoff = backoff + } +} + +// RetryableError is an interface that errors can implement to signal whether they are retryable. +type RetryableError interface { + error + IsRetryable() bool +} + +// Retry wraps a BlockingAsyncOp with a retry policy. +// It will retry the operation if it returns a types.Err that wraps a RetryableError returning true for IsRetryable. +func Retry(fn functions.BlockingAsyncOp, opts ...RetryOption) functions.BlockingAsyncOp { + config := &retryConfig{ + maxAttempts: 3, + backoff: 100 * time.Millisecond, + } + for _, opt := range opts { + opt(config) + } + + return func(ctx context.Context, args ...ref.Val) ref.Val { + var lastErr ref.Val + var backoff *time.Timer + defer func() { + if backoff != nil { + backoff.Stop() + } + }() + for i := 0; i < config.maxAttempts; i++ { + if i > 0 { + // Reuse a single timer across attempts and stop it on cancellation so the + // pending timer is not left to fire after the call returns. + if backoff == nil { + backoff = time.NewTimer(config.backoff) + } else { + backoff.Reset(config.backoff) + } + select { + case <-backoff.C: + case <-ctx.Done(): + backoff.Stop() + return types.NewErr("operation cancelled during retry: %v", ctx.Err()) + } + } + + res := fn(ctx, args...) + if !types.IsError(res) { + return res + } + + err := res.(*types.Err) + lastErr = res + + if !isRetryable(err) { + return res + } + } + return lastErr + } +} + +// RetryBinding wraps a BlockingAsyncOp with a retry policy and returns an OverloadOpt. +func RetryBinding(fn functions.BlockingAsyncOp, opts ...RetryOption) decls.OverloadOpt { + return decls.AsyncBinding(Retry(fn, opts...)) +} + +func isRetryable(err *types.Err) bool { + var re RetryableError + if errors.As(err, &re) { + return re.IsRetryable() + } + return false +} diff --git a/vendor/github.com/google/cel-go/cel/cel.go b/vendor/github.com/google/cel-go/cel/cel.go index eb5a9f4cc..9ba957a7e 100644 --- a/vendor/github.com/google/cel-go/cel/cel.go +++ b/vendor/github.com/google/cel-go/cel/cel.go @@ -17,3 +17,24 @@ // CEL is a non-Turing complete expression language designed to parse, check, and evaluate // expressions against user-defined environments. package cel + +// Compile is a convenience function that constructs a new Env using the provided EnvOption values, +// compiles the expression string, and plans an executable Program. +// +// Warning: Creating a new environment for every compilation is expensive. Environment setup should be done once +// and shared across expression compilations when the options remain the same. +func Compile(expression string, opts ...EnvOption) (Program, error) { + env, err := NewEnv(opts...) + if err != nil { + return nil, err + } + ast, iss := env.Compile(expression) + if iss.Err() != nil { + return nil, iss.Err() + } + prg, err := env.Program(ast, EvalOptions(OptOptimize)) + if err != nil { + return nil, err + } + return prg, nil +} diff --git a/vendor/github.com/google/cel-go/cel/decls.go b/vendor/github.com/google/cel-go/cel/decls.go index 4d4873bd6..c7c23fd51 100644 --- a/vendor/github.com/google/cel-go/cel/decls.go +++ b/vendor/github.com/google/cel-go/cel/decls.go @@ -346,6 +346,32 @@ func LateFunctionBinding() OverloadOpt { return decls.LateFunctionBinding() } +// AsyncBinding provides the implementation of an asynchronous overload. The provided function +// is called in its own goroutine with the provided context. The function should block until +// the result is available, and the framework manages goroutine and channel lifecycle. +// +// This follows the same pattern used by gRPC-Go and other major Go frameworks where user +// code is synchronous and the framework manages concurrency. +// +// Context contract: the function MUST return promptly once its context is cancelled. The +// framework cannot forcibly terminate the goroutine running the function, so a function that +// ignores cancellation will leak its goroutine and hold a concurrency slot (see +// AsyncMaxConcurrency) until it returns on its own. For functions that may hang or that are not +// under your control, wrap them with async.TimeoutBinding to bound their runtime. +func AsyncBinding(fn functions.BlockingAsyncOp) OverloadOpt { + return decls.AsyncBinding(fn) +} + +// SingletonAsyncBinding creates a singleton async function definition from a blocking function, +// to be used with all function overloads. The provided function is called in its own goroutine +// with the provided context. +// +// Note, this approach works well if operand is expected to have a specific trait which it implements, +// e.g. traits.ContainerType. Otherwise, prefer per-overload async bindings. +func SingletonAsyncBinding(fn functions.BlockingAsyncOp, traits ...int) FunctionOpt { + return decls.SingletonAsyncBinding(fn, traits...) +} + // OverloadIsNonStrict enables the function to be called with error and unknown argument values. // // Note: do not use this option unless absoluately necessary as it should be an uncommon feature. diff --git a/vendor/github.com/google/cel-go/cel/env.go b/vendor/github.com/google/cel-go/cel/env.go index e2de2ff6f..784790ba2 100644 --- a/vendor/github.com/google/cel-go/cel/env.go +++ b/vendor/github.com/google/cel-go/cel/env.go @@ -48,6 +48,10 @@ type Source = common.Source type Ast struct { source Source impl *celast.AST + // loadErr captures an error detected while loading the AST (e.g. an over-deep AST ingested via + // ParsedExprToAst / CheckedExprToAst) so it can be surfaced when the Ast is checked or planned + // instead of recursing into the checker or planner on adversarially deep input. + loadErr error } // NativeRep converts the AST to a Go-native representation. @@ -395,6 +399,20 @@ func NewCustomEnv(opts ...EnvOption) (*Env, error) { // It is possible to have both non-nil Ast and Issues values returned from this call: however, // the mere presence of an Ast does not imply that it is valid for use. func (e *Env) Check(ast *Ast) (*Ast, *Issues) { + // Surface any error recorded while the Ast was loaded (e.g. an over-deep AST rejected by + // ParsedExprToAst / CheckedExprToAst) before recursing into the type checker on it. + if ast != nil && ast.loadErr != nil { + errs := common.NewErrors(ast.Source()) + errs.ReportErrorString(common.NoLocation, ast.loadErr.Error()) + return nil, NewIssuesWithSourceInfo(errs, ast.NativeRep().SourceInfo()) + } + if nodeLimit := e.configuredExpressionNodeLimit(); nodeLimit > 0 && ast != nil && ast.NativeRep() != nil { + if count := celast.NodeCount(ast.NativeRep()); count > nodeLimit { + errs := common.NewErrors(ast.Source()) + errs.ReportErrorString(common.NoLocation, fmt.Sprintf("expression node count exceeds limit: count %d, limit %d", count, nodeLimit)) + return nil, NewIssuesWithSourceInfo(errs, ast.NativeRep().SourceInfo()) + } + } // Construct the internal checker env, erroring if there is an issue adding the declarations. chk, err := e.initChecker() if err != nil { @@ -436,6 +454,24 @@ func (e *Env) Check(ast *Ast) (*Ast, *Issues) { return ast, nil } +// configuredExpressionSizeLimit returns the effective expression size code point limit. +// A zero value means "use the parser default". +func (e *Env) configuredExpressionSizeLimit() int { + if l := e.limits[limitCodePointSize]; l != 0 { + return l + } + return 100_000 +} + +// configuredExpressionNodeLimit returns the effective expression node limit. +// A zero value means "use default". +func (e *Env) configuredExpressionNodeLimit() int { + if l := e.limits[limitExpressionNodeCount]; l != 0 { + return l + } + return 100_000 +} + // Compile combines the Parse and Check phases CEL program compilation to produce an Ast and // associated issues. // @@ -445,7 +481,11 @@ func (e *Env) Check(ast *Ast) (*Ast, *Issues) { // // Note, for parse-only uses of CEL use Parse. func (e *Env) Compile(txt string) (*Ast, *Issues) { - return e.CompileSource(common.NewTextSource(txt)) + src, err := common.NewTextSourceWithLimit(txt, e.configuredExpressionSizeLimit()) + if err != nil { + return nil, ErrorAsIssues(err) + } + return e.CompileSource(src) } // CompileSource combines the Parse and Check phases CEL program compilation to produce an Ast and @@ -650,7 +690,10 @@ func (e *Env) Validators() []ASTValidator { // This form of Parse creates a Source value for the input `txt` and forwards to the // ParseSource method. func (e *Env) Parse(txt string) (*Ast, *Issues) { - src := common.NewTextSource(txt) + src, err := common.NewTextSourceWithLimit(txt, e.configuredExpressionSizeLimit()) + if err != nil { + return nil, ErrorAsIssues(err) + } return e.ParseSource(src) } @@ -671,6 +714,12 @@ func (e *Env) ParseSource(src Source) (*Ast, *Issues) { // Program generates an evaluable instance of the Ast within the environment (Env). func (e *Env) Program(ast *Ast, opts ...ProgramOption) (Program, error) { + // Surface any error recorded while the Ast was loaded (e.g. an over-deep AST rejected by + // ParsedExprToAst / CheckedExprToAst) rather than recursing into the planner on it. This is a + // cheap field read; the depth traversal itself runs once at conversion time, not here. + if ast != nil && ast.loadErr != nil { + return nil, ast.loadErr + } return e.PlanProgram(ast.NativeRep(), opts...) } @@ -843,6 +892,9 @@ func (e *Env) configure(opts []EnvOption) (*Env, error) { if l := e.limits[limitParseRecursionDepth]; l != 0 { prsrOpts = append(prsrOpts, parser.MaxRecursionDepth(l)) } + if l := e.limits[limitExpressionNodeCount]; l != 0 { + prsrOpts = append(prsrOpts, parser.MaxExpressionNodeCount(l)) + } e.prsr, err = parser.NewParser(prsrOpts...) if err != nil { return nil, err diff --git a/vendor/github.com/google/cel-go/cel/folding.go b/vendor/github.com/google/cel-go/cel/folding.go index d1ea6b19d..5525f0805 100644 --- a/vendor/github.com/google/cel-go/cel/folding.go +++ b/vendor/github.com/google/cel-go/cel/folding.go @@ -15,6 +15,8 @@ package cel import ( + "context" + "errors" "fmt" "github.com/google/cel-go/common/ast" @@ -93,18 +95,18 @@ func (opt *constantFoldingOptimizer) Optimize(ctx *OptimizerContext, a *ast.AST) for _, fold := range foldableExprs { // If the expression could be folded because it's a non-strict call, and the // branches are pruned, continue to the next fold. - if fold.Kind() == ast.CallKind && maybePruneBranches(ctx, fold) { + if fold.Kind() == ast.CallKind && maybePruneBranches(ctx, a, fold) { continue } // Late-bound function calls cannot be folded. - if fold.Kind() == ast.CallKind && isLateBoundFunctionCall(ctx, a, fold) { + if fold.Kind() == ast.CallKind && isLateBoundFunctionCall(ctx, fold) { continue } // Otherwise, assume all context is needed to evaluate the expression. err := opt.tryFold(ctx, a, fold) - // Ignore errors for identifiers, since there is no guarantee that the environment + // Ignore errors for identifiers or subexpressions that cannot be folded, since there is no guarantee that the environment // has a value for them. - if err != nil && fold.Kind() != ast.IdentKind { + if err != nil && fold.Kind() != ast.IdentKind && !errors.Is(err, errCannotFold) { ctx.ReportErrorAtID(fold.ID(), "constant-folding evaluation failed: %v", err.Error()) return a } @@ -142,24 +144,19 @@ func (opt *constantFoldingOptimizer) Optimize(ctx *OptimizerContext, a *ast.AST) return a } +var errCannotFold = errors.New("subexpression cannot be folded") + // tryFold attempts to evaluate a sub-expression to a literal. // // If the evaluation succeeds, the input expr value will be modified to become a literal, otherwise // the method will return an error. func (opt *constantFoldingOptimizer) tryFold(ctx *OptimizerContext, a *ast.AST, expr ast.Expr) error { - // Assume all context is needed to evaluate the expression. - subAST := &Ast{ - impl: ast.NewCheckedAST(ast.NewAST(expr, a.SourceInfo()), a.TypeMap(), a.ReferenceMap()), - } - prg, err := ctx.Program(subAST) - if err != nil { - return err - } activation := opt.knownValues if activation == nil { activation = NoVars() } - out, _, err := prg.Eval(activation) + navExpr := expr.(ast.NavigableExpr) + out, err := evaluateExpr(ctx, a, navExpr, activation) if err != nil { return err } @@ -168,7 +165,31 @@ func (opt *constantFoldingOptimizer) tryFold(ctx *OptimizerContext, a *ast.AST, return nil } -func isLateBoundFunctionCall(ctx *OptimizerContext, a *ast.AST, expr ast.Expr) bool { +func evaluateExpr(ctx *OptimizerContext, a *ast.AST, navigableExpr ast.NavigableExpr, activation Activation) (ref.Val, error) { + partialActivation, err := ctx.PartialVars(activation) + if err != nil { + return nil, err + } + subAST := &Ast{ + impl: ast.NewCheckedAST(ast.NewAST(navigableExpr, a.SourceInfo()), a.TypeMap(), a.ReferenceMap()), + } + prg, err := ctx.Program(subAST) + if err != nil { + return nil, err + } + // Folding will not attempt to call async functions which are all marked as late-bound, + // but the presence of such functions requires the use of `ConcurrentEval` in order to + // avoid an early return error which blocks async functions from running in `Eval` and + // `ContextEval` call paths. + resCh := prg.ConcurrentEval(context.Background(), partialActivation) + res := <-resCh + if res.Err != nil || types.IsUnknown(res.Val) { + return nil, errCannotFold + } + return res.Val, nil +} + +func isLateBoundFunctionCall(ctx *OptimizerContext, expr ast.Expr) bool { call := expr.AsCall() function := ctx.Functions()[call.FunctionName()] if function == nil { @@ -181,12 +202,12 @@ func isLateBoundFunctionCall(ctx *OptimizerContext, a *ast.AST, expr ast.Expr) b // a branch can be removed. Evaluation will naturally prune logical and / or calls, // but conditional will not be pruned cleanly, so this is one small area where the // constant folding step reimplements a portion of the evaluator. -func maybePruneBranches(ctx *OptimizerContext, expr ast.NavigableExpr) bool { +func maybePruneBranches(ctx *OptimizerContext, a *ast.AST, expr ast.NavigableExpr) bool { call := expr.AsCall() args := call.Args() switch call.FunctionName() { case operators.LogicalAnd, operators.LogicalOr: - return maybeShortcircuitLogic(ctx, call.FunctionName(), args, expr) + return maybeShortcircuitLogic(ctx, a, call.FunctionName(), args, expr) case operators.Conditional: cond := args[0] truthy := args[1] @@ -207,11 +228,17 @@ func maybePruneBranches(ctx *OptimizerContext, expr ast.NavigableExpr) bool { return true } needle := args[0] - if needle.Kind() == ast.LiteralKind && haystack.Kind() == ast.ListKind { - needleValue := needle.AsLiteral() + if (needle.Kind() == ast.LiteralKind || isSelfEqualIdent(needle)) && haystack.Kind() == ast.ListKind { + needleIsLit := needle.Kind() == ast.LiteralKind + needleLitVal := needle.AsLiteral() + needleIdentVal := needle.AsIdent() list := haystack.AsList() - for _, e := range list.Elements() { - if e.Kind() == ast.LiteralKind && e.AsLiteral().Equal(needleValue) == types.True { + for _, elem := range list.Elements() { + if needleIsLit && elem.Kind() == ast.LiteralKind && elem.AsLiteral().Equal(needleLitVal) == types.True { + ctx.UpdateExpr(expr, ctx.NewLiteral(types.True)) + return true + } + if !needleIsLit && elem.Kind() == ast.IdentKind && elem.AsIdent() == needleIdentVal { ctx.UpdateExpr(expr, ctx.NewLiteral(types.True)) return true } @@ -221,7 +248,7 @@ func maybePruneBranches(ctx *OptimizerContext, expr ast.NavigableExpr) bool { return false } -func maybeShortcircuitLogic(ctx *OptimizerContext, function string, args []ast.Expr, expr ast.NavigableExpr) bool { +func maybeShortcircuitLogic(ctx *OptimizerContext, a *ast.AST, function string, args []ast.Expr, expr ast.NavigableExpr) bool { shortcircuit := types.False skip := types.True if function == operators.LogicalOr { @@ -244,10 +271,14 @@ func maybeShortcircuitLogic(ctx *OptimizerContext, function string, args []ast.E } if len(newArgs) == 0 { newArgs = append(newArgs, args[0]) - ctx.UpdateExpr(expr, newArgs[0]) - return true + } + if len(newArgs) == len(args) { + return false } if len(newArgs) == 1 { + if !isBoolType(a, newArgs[0]) { + return false + } ctx.UpdateExpr(expr, newArgs[0]) return true } @@ -255,6 +286,16 @@ func maybeShortcircuitLogic(ctx *OptimizerContext, function string, args []ast.E return true } +func isBoolType(a *ast.AST, e ast.Expr) bool { + if a != nil && a.GetType(e.ID()) == types.BoolType { + return true + } + if e.Kind() == ast.LiteralKind && e.AsLiteral().Type() == types.BoolType { + return true + } + return false +} + // pruneOptionalElements works from the bottom up to resolve optional elements within // aggregate literals. // @@ -285,9 +326,9 @@ func pruneOptionalListElements(ctx *OptimizerContext, e ast.Expr) { updatedElems := []ast.Expr{} updatedIndices := []int32{} newOptIndex := -1 - for _, e := range elems { + for i, e := range elems { newOptIndex++ - if !l.IsOptional(int32(newOptIndex)) { + if !l.IsOptional(int32(i)) { updatedElems = append(updatedElems, e) continue } @@ -501,7 +542,7 @@ func (opt *constantFoldingOptimizer) constantExprMatcher(ctx *OptimizerContext, sel := e.AsSelect() // guaranteed to be a navigable value return constantMatcher(sel.Operand().(ast.NavigableExpr)) case ast.IdentKind: - return opt.knownValues != nil && a.ReferenceMap()[e.ID()] != nil + return opt.knownValues != nil && a.ReferenceMap()[e.ID()] != nil && !hasComprehensionVar(e) case ast.ComprehensionKind: if isNestedComprehension(e) { return false @@ -513,12 +554,15 @@ func (opt *constantFoldingOptimizer) constantExprMatcher(ctx *OptimizerContext, nested := e.AsComprehension() vars[nested.AccuVar()] = true vars[nested.IterVar()] = true + if nested.IterVar2() != "" { + vars[nested.IterVar2()] = true + } } if e.Kind() == ast.IdentKind && !vars[e.AsIdent()] { constantExprs = false } // Late-bound function calls cannot be folded. - if e.Kind() == ast.CallKind && isLateBoundFunctionCall(ctx, a, e) { + if e.Kind() == ast.CallKind && isLateBoundFunctionCall(ctx, e) { constantExprs = false } }) @@ -554,17 +598,33 @@ func constantCallMatcher(e ast.NavigableExpr) bool { return true } } + if fnName == operators.Equals || fnName == operators.NotEquals { + if hasComprehensionVar(e) { + return false + } + if isExprConstantOfKind(children[0], types.BoolType) || isExprConstantOfKind(children[1], types.BoolType) { + return true + } + } if fnName == operators.In { + if hasComprehensionVar(e) { + return false + } haystack := children[1] if haystack.Kind() == ast.ListKind && haystack.AsList().Size() == 0 { return true } needle := children[0] - if needle.Kind() == ast.LiteralKind && haystack.Kind() == ast.ListKind { - needleValue := needle.AsLiteral() + if (needle.Kind() == ast.LiteralKind || isSelfEqualIdent(needle)) && haystack.Kind() == ast.ListKind { + needleIsLit := needle.Kind() == ast.LiteralKind + needleLitVal := needle.AsLiteral() + needleIdentVal := needle.AsIdent() list := haystack.AsList() - for _, e := range list.Elements() { - if e.Kind() == ast.LiteralKind && e.AsLiteral().Equal(needleValue) == types.True { + for _, elem := range list.Elements() { + if needleIsLit && elem.Kind() == ast.LiteralKind && elem.AsLiteral().Equal(needleLitVal) == types.True { + return true + } + if !needleIsLit && elem.Kind() == ast.IdentKind && elem.AsIdent() == needleIdentVal { return true } } @@ -579,6 +639,74 @@ func constantCallMatcher(e ast.NavigableExpr) bool { return true } +// isSelfEqualIdent indicates whether the expression is an identifier whose static type +// guarantees that its runtime value is equal to itself. +// +// Matching an identifier against a list element by name only proves list membership when the +// value the name resolves to is self-equal. A double may be NaN, which is not equal to itself, +// and dyn, abstract, and struct types may all hold a NaN at runtime, so the check is limited +// to the scalar types which cannot, and to the aggregate types whose type parameters are +// themselves self-equal. +func isSelfEqualIdent(e ast.Expr) bool { + if e.Kind() != ast.IdentKind { + return false + } + nav, ok := e.(ast.NavigableExpr) + if !ok { + return false + } + return isSelfEqualType(nav.Type()) +} + +// isSelfEqualType indicates whether all runtime values of the given type are equal to themselves. +func isSelfEqualType(t *types.Type) bool { + if t == nil { + return false + } + switch t.Kind() { + case types.BoolKind, types.BytesKind, types.DurationKind, types.IntKind, + types.NullTypeKind, types.StringKind, types.TimestampKind, types.TypeKind, + types.UintKind: + return true + case types.ListKind, types.MapKind: + // Aggregates compare element-wise, so they are self-equal exactly when their type + // parameters are. A list(dyn) or map(string, double) may still contain a NaN. + for _, p := range t.Parameters() { + if !isSelfEqualType(p) { + return false + } + } + return true + default: + return false + } +} + +func isExprConstantOfKind(e ast.Expr, t *types.Type) bool { + return e.Kind() == ast.LiteralKind && e.AsLiteral().Type() == t +} + +func hasComprehensionVar(e ast.NavigableExpr) bool { + idents := ast.MatchDescendants(e, ast.KindMatcher(ast.IdentKind)) + for _, identNode := range idents { + identName := identNode.AsIdent() + curr := identNode + parent, found := curr.Parent() + for found { + if parent.Kind() == ast.ComprehensionKind { + compre := parent.AsComprehension() + if (compre.AccuVar() == identName || compre.IterVar() == identName || compre.IterVar2() == identName) && + curr.ID() != compre.IterRange().ID() && curr.ID() != compre.AccuInit().ID() { + return true + } + } + curr = parent + parent, found = parent.Parent() + } + } + return false +} + func isNestedComprehension(e ast.NavigableExpr) bool { parent, found := e.Parent() for found { diff --git a/vendor/github.com/google/cel-go/cel/io.go b/vendor/github.com/google/cel-go/cel/io.go index 2e611228d..c991c95c3 100644 --- a/vendor/github.com/google/cel-go/cel/io.go +++ b/vendor/github.com/google/cel-go/cel/io.go @@ -52,7 +52,12 @@ func CheckedExprToAstWithSource(checkedExpr *exprpb.CheckedExpr, src Source) (*A if err != nil { return nil, err } - return &Ast{source: src, impl: checked}, nil + out := &Ast{source: src, impl: checked} + if err := checkLoadedASTDepth(checked); err != nil { + out.loadErr = err + return out, err + } + return out, nil } // AstToCheckedExpr converts an Ast to an protobuf CheckedExpr value. @@ -83,7 +88,26 @@ func ParsedExprToAstWithSource(parsedExpr *exprpb.ParsedExpr, src Source) *Ast { src = common.NewInfoSource(parsedExpr.GetSourceInfo()) } e, _ := ast.ProtoToExpr(parsedExpr.GetExpr()) - return &Ast{source: src, impl: ast.NewAST(e, info)} + out := &Ast{source: src, impl: ast.NewAST(e, info)} + // ParsedExprToAstWithSource has no error return, so record an over-depth violation on the Ast + // to be surfaced when it is later checked or planned. + out.loadErr = checkLoadedASTDepth(out.impl) + return out +} + +// checkLoadedASTDepth guards ASTs that enter through the proto conversion helpers +// (ParsedExprToAst / CheckedExprToAst) against nesting deeper than the parser's recursion limit. +// Those entry points bypass the parser, so without this check a deeply nested loaded AST could +// exhaust the Go stack during later checking or planning. It returns a normal error rather than +// risking that overflow; the traversal itself is bounded so it stays safe on the same input. +// +// Embedders that fully control their AST inputs can skip this by building the AST through the +// common/ast package directly instead of these conversion helpers. +func checkLoadedASTDepth(a *ast.AST) error { + if ast.ExceedsDepth(a, defaultMaxASTDepth) { + return fmt.Errorf("input exceeds maximum expression nesting depth: %d", defaultMaxASTDepth) + } + return nil } // AstToParsedExpr converts an Ast to an protobuf ParsedExpr value. diff --git a/vendor/github.com/google/cel-go/cel/library.go b/vendor/github.com/google/cel-go/cel/library.go index 3c8b6ba34..332eb3f17 100644 --- a/vendor/github.com/google/cel-go/cel/library.go +++ b/vendor/github.com/google/cel-go/cel/library.go @@ -590,7 +590,7 @@ func (lib *optionalLib) CompileOptions() []EnvOption { // ProgramOptions implements the Library interface method. func (lib *optionalLib) ProgramOptions() []ProgramOption { return []ProgramOption{ - CustomDecorator(decorateOptionalOr), + CustomDecoratorV2(decorateOptionalOr), } } @@ -683,7 +683,7 @@ func EnableErrorOnBadPresenceTest(value bool) EnvOption { return features(featureEnableErrorOnBadPresenceTest, value) } -func decorateOptionalOr(i interpreter.Interpretable) (interpreter.Interpretable, error) { +func decorateOptionalOr(i interpreter.InterpretableV2) (interpreter.InterpretableV2, error) { call, ok := i.(interpreter.InterpretableCall) if !ok { return i, nil @@ -720,8 +720,8 @@ func decorateOptionalOr(i interpreter.Interpretable) (interpreter.Interpretable, // the second optional expression is evaluated and returned. type evalOptionalOr struct { id int64 - lhs interpreter.Interpretable - rhs interpreter.Interpretable + lhs interpreter.InterpretableV2 + rhs interpreter.InterpretableV2 } // ID implements the Interpretable interface method. @@ -729,11 +729,9 @@ func (opt *evalOptionalOr) ID() int64 { return opt.id } -// Eval evaluates the left-hand side optional to determine whether it contains a value, else -// proceeds with the right-hand side evaluation. -func (opt *evalOptionalOr) Eval(ctx interpreter.Activation) ref.Val { +func (opt *evalOptionalOr) Exec(frame *interpreter.ExecutionFrame) ref.Val { // short-circuit lhs. - optLHS := opt.lhs.Eval(ctx) + optLHS := opt.lhs.Exec(frame) switch val := optLHS.(type) { case *types.Err, *types.Unknown: return optLHS @@ -741,18 +739,24 @@ func (opt *evalOptionalOr) Eval(ctx interpreter.Activation) ref.Val { if val.HasValue() { return optLHS } - return opt.rhs.Eval(ctx) + return opt.rhs.Exec(frame) default: return types.NoSuchOverloadErr() } } +// Eval evaluates the left-hand side optional to determine whether it contains a value, else +// proceeds with the right-hand side evaluation. +func (opt *evalOptionalOr) Eval(ctx interpreter.Activation) ref.Val { + return opt.Exec(interpreter.AsFrame(ctx)) +} + // evalOptionalOrValue selects between an optional or a concrete value. If the optional has a value, // its value is returned, otherwise the alternative value expression is evaluated and returned. type evalOptionalOrValue struct { id int64 - lhs interpreter.Interpretable - rhs interpreter.Interpretable + lhs interpreter.InterpretableV2 + rhs interpreter.InterpretableV2 } // ID implements the Interpretable interface method. @@ -760,11 +764,9 @@ func (opt *evalOptionalOrValue) ID() int64 { return opt.id } -// Eval evaluates the left-hand side optional to determine whether it contains a value, else -// proceeds with the right-hand side evaluation. -func (opt *evalOptionalOrValue) Eval(ctx interpreter.Activation) ref.Val { +func (opt *evalOptionalOrValue) Exec(frame *interpreter.ExecutionFrame) ref.Val { // short-circuit lhs. - optLHS := opt.lhs.Eval(ctx) + optLHS := opt.lhs.Exec(frame) switch val := optLHS.(type) { case *types.Err, *types.Unknown: @@ -773,12 +775,18 @@ func (opt *evalOptionalOrValue) Eval(ctx interpreter.Activation) ref.Val { if val.HasValue() { return val.GetValue() } - return opt.rhs.Eval(ctx) + return opt.rhs.Exec(frame) default: return types.NoSuchOverloadErr() } } +// Eval evaluates the left-hand side optional to determine whether it contains a value, else +// proceeds with the right-hand side evaluation. +func (opt *evalOptionalOrValue) Eval(ctx interpreter.Activation) ref.Val { + return opt.Exec(interpreter.AsFrame(ctx)) +} + type timeLegacyLibrary struct{} func (timeLegacyLibrary) CompileOptions() []EnvOption { diff --git a/vendor/github.com/google/cel-go/cel/options.go b/vendor/github.com/google/cel-go/cel/options.go index d7d2ab034..540ad38ba 100644 --- a/vendor/github.com/google/cel-go/cel/options.go +++ b/vendor/github.com/google/cel-go/cel/options.go @@ -24,6 +24,7 @@ import ( "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/dynamicpb" + "github.com/google/cel-go/cel/async" "github.com/google/cel-go/checker" "github.com/google/cel-go/common/containers" "github.com/google/cel-go/common/decls" @@ -109,12 +110,23 @@ const ( limitCodePointSize // The number of attempts to recover from a parse error. limitParseErrorRecovery + // The maximum nesting depth permitted for ASTs loaded outside the parser. + limitMaxASTDepth + // The maximum number of expression nodes permitted in parsing (including macro expansion). + limitExpressionNodeCount ) +// defaultMaxASTDepth mirrors the parser's default maxRecursionDepth (250) and +// is applied to ASTs that enter through non-parser ingestion paths (e.g. via +// ParsedExprToAst / CheckedExprToAst) when no explicit limit is configured. +const defaultMaxASTDepth = 250 + var limitIDsToNames = map[limitID]string{ limitCodePointSize: "cel.limit.expression_code_points", limitParseErrorRecovery: "cel.limit.parse_error_recovery", limitParseRecursionDepth: "cel.limit.parse_recursion_depth", + limitMaxASTDepth: "cel.limit.max_ast_depth", + limitExpressionNodeCount: "cel.limit.expression_node_count", } func limitNameByID(id limitID) (string, bool) { @@ -456,6 +468,14 @@ func CustomDecorator(dec interpreter.InterpretableDecorator) ProgramOption { } } +// CustomDecoratorV2 appends an InterpreterDecoratorV2 to the program. +func CustomDecoratorV2(dec interpreter.InterpretableDecoratorV2) ProgramOption { + return func(p *prog) (*prog, error) { + p.plannerOptions = append(p.plannerOptions, interpreter.CustomDecoratorV2(dec)) + return p, nil + } +} + // Functions adds function overloads that extend or override the set of CEL built-ins. // // Deprecated: use Function() instead to declare the function, its overload signatures, @@ -727,6 +747,47 @@ func InterruptCheckFrequency(checkFrequency uint) ProgramOption { } } +// AsyncCallObserver sets the observer for monitoring asynchronous function calls during ConcurrentEval. +func AsyncCallObserver(observer async.Observer) ProgramOption { + return func(p *prog) (*prog, error) { + p.asyncObserver = observer + return p, nil + } +} + +// AsyncCompletionBufferSize sets the size of the buffer for the async completion channel. +// By default, the channel is unbuffered. +func AsyncCompletionBufferSize(size int) ProgramOption { + return func(p *prog) (*prog, error) { + p.asyncCompletionBufferSize = size + return p, nil + } +} + +// AsyncMaxConcurrency sets the maximum number of concurrently launched async calls during +// ConcurrentEval. This bounds the number of in-flight async goroutines, so a wide fan-out (such +// as an async call inside a comprehension over a large list) cannot exhaust memory. +// +// A value of 0 (unset) applies a built-in default bound. A positive value sets an explicit bound. +// A negative value disables the limiter (unbounded launches) and should only be used when +// concurrency is bounded by other means. +func AsyncMaxConcurrency(maxConcurrency int) ProgramOption { + return func(p *prog) (*prog, error) { + p.asyncMaxConcurrency = maxConcurrency + return p, nil + } +} + +// ConcurrentDrainStrategy configures the strategy for when to re-evaluate the program +// during a ConcurrentEval call after receiving asynchronous completion signals. +// By default, the program re-evaluates immediately after every completion. +func ConcurrentDrainStrategy(strategy async.DrainStrategy) ProgramOption { + return func(p *prog) (*prog, error) { + p.drainStrategy = strategy + return p, nil + } +} + // CostEstimatorOptions configure type-check time options for estimating expression cost. func CostEstimatorOptions(costOpts ...checker.CostOption) EnvOption { return func(e *Env) (*Env, error) { @@ -942,6 +1003,25 @@ func ParserExpressionSizeLimit(limit int) EnvOption { return setLimit(limitCodePointSize, limit) } +// ExpressionNodeLimit adjusts the maximum number of expression nodes permitted during parsing +// and checking, including nodes created by macro expansion. Defaults are defined in the parser +// package (100,000). A negative value means unbounded. +func ExpressionNodeLimit(limit int) EnvOption { + return setLimit(limitExpressionNodeCount, limit) +} + +// ExpressionNestingDepthLimit records the maximum nesting depth permitted for ASTs in the +// environment configuration so that the value round-trips through env.Config export/import. +// +// ASTs loaded outside the parser (e.g. via ParsedExprToAst / CheckedExprToAst) bypass the +// parser's recursion limit, so those conversion paths validate nesting depth against the +// parser-matching default (250) to avoid a Go stack overflow during later checking or planning. +// Embedders that fully control their AST inputs and want to skip the check can construct the AST +// through the common/ast package directly rather than the cel conversion helpers. +func ExpressionNestingDepthLimit(limit int) EnvOption { + return setLimit(limitMaxASTDepth, limit) +} + // EnableHiddenAccumulatorName sets the parser to use the identifier '@result' for accumulators // which is not normally accessible from CEL source. func EnableHiddenAccumulatorName(enabled bool) EnvOption { diff --git a/vendor/github.com/google/cel-go/cel/program.go b/vendor/github.com/google/cel-go/cel/program.go index c46d694e4..3a7589a71 100644 --- a/vendor/github.com/google/cel-go/cel/program.go +++ b/vendor/github.com/google/cel-go/cel/program.go @@ -18,8 +18,9 @@ import ( "context" "errors" "fmt" - "sync" + "time" + "github.com/google/cel-go/cel/async" "github.com/google/cel-go/common/ast" "github.com/google/cel-go/common/functions" "github.com/google/cel-go/common/types" @@ -53,6 +54,21 @@ type Program interface { // // The output contract for `ContextEval` is otherwise identical to the `Eval` method. ContextEval(context.Context, any) (ref.Val, *EvalDetails, error) + + // ConcurrentEval evaluates the program concurrently, returning a channel that will receive + // the final EvalResult when all asynchronous operations complete, or the context expires. + // + // The vars value may either be an `Activation` or `map[string]any`. + // + // Liveness: ConcurrentEval relies on context cancellation to terminate. If an async function + // never returns and does not honor its context, and the supplied context has no deadline, the + // call will block indefinitely. Always pass a context with a deadline or cancellation. + // + // Error handling is fail-fast: as soon as a re-evaluation pass yields an error, that error is + // returned and any still in-flight async calls are cancelled (their contexts are done) and + // their results discarded. Async functions should therefore be free of unwanted side effects + // on partial evaluation, or guard them with idempotency/cancellation handling. + ConcurrentEval(context.Context, any) <-chan EvalResult } // Activation used to resolve identifiers by name and references by id. @@ -145,6 +161,13 @@ func (ed *EvalDetails) ActualCost() *uint64 { return &cost } +// EvalResult encapsulates the response from a ConcurrentEval call. +type EvalResult struct { + Val ref.Val + EvalDetails *EvalDetails + Err error +} + // prog is the internal implementation of the Program interface. type prog struct { *Env @@ -160,11 +183,21 @@ type prog struct { regexOptimizations []*interpreter.RegexOptimization // Interpretable configured from an Ast and aggregate decorator set based on program options. - interpretable interpreter.Interpretable + interpretable interpreter.InterpretableV2 observable *interpreter.ObservableInterpretable callCostEstimator interpreter.ActualCostEstimator costOptions []interpreter.CostTrackerOption costLimit *uint64 + + // hasAsync indicates the planned expression contains an asynchronous function call, which can + // only be resolved by ConcurrentEval. + hasAsync bool + + // Async evaluation configuration used by ConcurrentEval. + drainStrategy async.DrainStrategy + asyncObserver async.Observer + asyncCompletionBufferSize int + asyncMaxConcurrency int } // newProgram creates a program instance with an environment, an ast, and an optional list of @@ -182,6 +215,7 @@ func newProgram(e *Env, a *ast.AST, opts []ProgramOption) (Program, error) { plannerOptions: []interpreter.PlannerOption{}, dispatcher: disp, costOptions: []interpreter.CostTrackerOption{}, + drainStrategy: async.DrainReady(100 * time.Microsecond), } // Configure the program via the ProgramOption values. @@ -214,6 +248,17 @@ func newProgram(e *Env, a *ast.AST, opts []ProgramOption) (Program, error) { return nil, err } + // Determine whether the environment declares any asynchronous function. Async is a property of + // the binding, so its presence is known from the environment alone, without inspecting the + // program plan. The synchronous entry points (Eval, ContextEval) reject programs from an env + // with async functions; callers needing synchronous evaluation should use a non-async env. + for _, b := range e.functionBindings { + if b.Async != nil { + p.hasAsync = true + break + } + } + // Set the attribute factory after the options have been set. var attrFactory interpreter.AttributeFactory attrFactorOpts := []interpreter.AttrFactoryOption{ @@ -262,8 +307,16 @@ func newProgram(e *Env, a *ast.AST, opts []ProgramOption) (Program, error) { if p.costLimit != nil { costOpts = append(costOpts, interpreter.CostTrackerLimit(*p.costLimit)) } + // Creating a new cost tracker for each evaluation causes significant work that + // needs to be repeated for each evaluation even though the cost tracker is + // mostly read-only once constructed. Therefore it gets constructed + // once now and later a cheap clone is used for each evaluation. + tracker, err := interpreter.NewCostTracker(p.callCostEstimator, costOpts...) + if err != nil { + return nil, fmt.Errorf("construct cost tracker: %w", err) + } trackerFactory := func() (*interpreter.CostTracker, error) { - return interpreter.NewCostTracker(p.callCostEstimator, costOpts...) + return tracker.Clone() } var observers []interpreter.PlannerOption if p.evalOpts&(OptExhaustiveEval|OptTrackState) != 0 { @@ -312,23 +365,25 @@ func (p *prog) Eval(input any) (out ref.Val, det *EvalDetails, err error) { } } }() - // Build a hierarchical activation if there are default vars set. - var vars Activation - switch v := input.(type) { - case Activation: - vars = v - case map[string]any: - vars = activationPool.Setup(v) - defer activationPool.Put(vars) - default: - return nil, nil, fmt.Errorf("invalid input, wanted Activation or map[string]any, got: (%T)%v", input, input) + // Asynchronous calls cannot be resolved by a single-pass evaluation. Reject before doing any + // work (this also covers ContextEval, which delegates here); ConcurrentEval does not call Eval. + if p.hasAsync { + return nil, nil, errAsyncRequiresConcurrentEval } - if p.defaultVars != nil { - vars = interpreter.NewHierarchicalActivation(p.defaultVars, vars) + // Build a hierarchical activation if there are default vars set. + var frame *interpreter.ExecutionFrame + if f, ok := input.(*interpreter.ExecutionFrame); ok { + frame = f + } else { + frame, err = p.newExecutionFrame(input) + if err != nil { + return nil, nil, err + } + defer frame.Close() } if p.observable != nil { det = &EvalDetails{} - out = p.observable.ObserveEval(vars, func(observed any) { + out = p.observable.ObserveExec(frame, func(observed any) { switch o := observed.(type) { case interpreter.EvalState: det.state = o @@ -337,7 +392,7 @@ func (p *prog) Eval(input any) (out ref.Val, det *EvalDetails, err error) { } }) } else { - out = p.interpretable.Eval(vars) + out = p.interpretable.Exec(frame) } // The output of an internal Eval may have a value (`v`) that is a types.Err. This step // translates the CEL value to a Go error response. This interface does not quite match the @@ -353,164 +408,220 @@ func (p *prog) ContextEval(ctx context.Context, input any) (ref.Val, *EvalDetail if ctx == nil { return nil, nil, fmt.Errorf("context can not be nil") } - // Configure the input, making sure to wrap Activation inputs in the special ctxActivation which - // exposes the #interrupted variable and manages rate-limited checks of the ctx.Done() state. - var vars Activation - switch v := input.(type) { - case Activation: - vars = ctxActivationPool.Setup(v, ctx.Done(), p.interruptCheckFrequency) - defer ctxActivationPool.Put(vars) - case map[string]any: - rawVars := activationPool.Setup(v) - defer activationPool.Put(rawVars) - vars = ctxActivationPool.Setup(rawVars, ctx.Done(), p.interruptCheckFrequency) - defer ctxActivationPool.Put(vars) - default: - return nil, nil, fmt.Errorf("invalid input, wanted Activation or map[string]any, got: (%T)%v", input, input) + frame, err := p.newExecutionFrame(input) + if err != nil { + return nil, nil, err } - out, det, err := p.Eval(vars) - if err != nil && errors.Is(err, interpreter.InterruptError{}) { - return out, det, context.Cause(ctx) + defer frame.Close() + frame.SetContext(ctx, p.interruptCheckFrequency) + out, det, errEval := p.Eval(frame) + if errEval != nil && errors.Is(errEval, interpreter.InterruptError{}) { + return out, det, fmt.Errorf("%w: %w", errEval, context.Cause(ctx)) } - return out, det, err + return out, det, errEval } -type ctxEvalActivation struct { - parent Activation - interrupt <-chan struct{} - interruptCheckCount uint - interruptCheckFrequency uint -} - -// ResolveName implements the Activation interface method, but adds a special #interrupted variable -// which is capable of testing whether a 'done' signal is provided from a context.Context channel. -func (a *ctxEvalActivation) ResolveName(name string) (any, bool) { - if name == "#interrupted" { - a.interruptCheckCount++ - if a.interruptCheckCount%a.interruptCheckFrequency == 0 { - select { - case <-a.interrupt: - return true, true - default: - return nil, false - } - } - return nil, false +// newExecutionFrame creates an ExecutionFrame for the given input without a timeout context. +func (p *prog) newExecutionFrame(input any) (*interpreter.ExecutionFrame, error) { + frame, err := interpreter.NewExecutionFrame(input) + if err != nil { + return nil, err + } + if p.defaultVars != nil { + // Update the frame's activation in place. + frame.Activation = interpreter.NewHierarchicalActivation(p.defaultVars, frame.Activation) } - return a.parent.ResolveName(name) -} -func (a *ctxEvalActivation) Parent() Activation { - return a.parent + return frame, nil } -func (a *ctxEvalActivation) AsPartialActivation() (interpreter.PartialActivation, bool) { - pa, ok := a.parent.(interpreter.PartialActivation) - return pa, ok +// newAsyncFrame creates an ExecutionFrame configured for asynchronous evaluation under the +// given context, wiring the observer and concurrency limit from the program options. +func (p *prog) newAsyncFrame(ctx context.Context, input any) (*interpreter.ExecutionFrame, error) { + frame, err := p.newExecutionFrame(input) + if err != nil { + return nil, err + } + if err := frame.SetContext(ctx, p.interruptCheckFrequency); err != nil { + frame.Close() + return nil, err + } + frame.SetAsyncObserver(p.asyncObserver) + frame.SetAsyncMaxConcurrency(resolveAsyncMaxConcurrency(p.asyncMaxConcurrency)) + return frame, nil } -func newCtxEvalActivationPool() *ctxEvalActivationPool { - return &ctxEvalActivationPool{ - Pool: sync.Pool{ - New: func() any { - return &ctxEvalActivation{} - }, - }, +// defaultAsyncMaxConcurrency bounds the number of concurrently launched async calls when the +// program does not configure AsyncMaxConcurrency. It exists so that a wide fan-out (e.g. an async +// call inside a comprehension over a large list) cannot spawn an unbounded number of goroutines. +const defaultAsyncMaxConcurrency = 100 + +// resolveAsyncMaxConcurrency maps the configured concurrency to the effective launch limit: +// - 0 (unset): apply defaultAsyncMaxConcurrency. +// - >0: use the configured value. +// - <0: unlimited (no launch limiter); use only if the caller bounds concurrency another way. +func resolveAsyncMaxConcurrency(configured int) int { + if configured == 0 { + return defaultAsyncMaxConcurrency } + return configured } -type ctxEvalActivationPool struct { - sync.Pool +// resolveCompletionBufferSize returns the size of the async completion channel. When unset, it +// defaults to the effective launch concurrency so that all in-flight calls can report completion +// without blocking. An unbuffered channel would make a completed call hold its launch slot until +// the evaluator drained it, throttling effective concurrency to the drain rate. +func (p *prog) resolveCompletionBufferSize() int { + if p.asyncCompletionBufferSize > 0 { + return p.asyncCompletionBufferSize + } + limit := resolveAsyncMaxConcurrency(p.asyncMaxConcurrency) + if limit < 0 { + // Unlimited launches: fall back to the default bound for the buffer so it stays finite. + return defaultAsyncMaxConcurrency + } + return limit } -// Setup initializes a pooled Activation with the ability check for context.Context cancellation -func (p *ctxEvalActivationPool) Setup(vars Activation, done <-chan struct{}, interruptCheckRate uint) *ctxEvalActivation { - a := p.Pool.Get().(*ctxEvalActivation) - a.parent = vars - a.interrupt = done - a.interruptCheckCount = 0 - a.interruptCheckFrequency = interruptCheckRate - return a -} +// ConcurrentEval implements the Program interface. +func (p *prog) ConcurrentEval(ctx context.Context, input any) <-chan EvalResult { + resCh := make(chan EvalResult, 1) + if ctx == nil { + resCh <- EvalResult{Err: errors.New("context can not be nil")} + close(resCh) + return resCh + } -type evalActivation struct { - vars map[string]any - lazyVars map[string]any -} + go func() { + defer close(resCh) + // Ensure concurrent eval handles panic / recovery properly + defer func() { + if r := recover(); r != nil { + switch t := r.(type) { + case interpreter.EvalCancelledError: + resCh <- EvalResult{Err: t} + default: + resCh <- EvalResult{Err: fmt.Errorf("internal error: %v", r)} + } + } + }() -// ResolveName looks up the value of the input variable name, if found. -// -// Lazy bindings may be supplied within the map-based input in either of the following forms: -// - func() any -// - func() ref.Val -// -// The lazy binding will only be invoked once per evaluation. -// -// Values which are not represented as ref.Val types on input may be adapted to a ref.Val using -// the types.Adapter configured in the environment. -func (a *evalActivation) ResolveName(name string) (any, bool) { - v, found := a.vars[name] - if !found { - return nil, false - } - switch obj := v.(type) { - case func() ref.Val: - if resolved, found := a.lazyVars[name]; found { - return resolved, true - } - lazy := obj() - a.lazyVars[name] = lazy - return lazy, true - case func() any: - if resolved, found := a.lazyVars[name]; found { - return resolved, true + frame, err := p.newAsyncFrame(ctx, input) + if err != nil { + resCh <- EvalResult{Err: err} + return } - lazy := obj() - a.lazyVars[name] = lazy - return lazy, true - default: - return obj, true - } -} + defer frame.Close() + + // Completions are signaled to this channel as async calls finish. The asyncCallState + // fan-in also selects on ctx.Done(), so the sender will not leak if this loop returns early. + completions := make(chan int64, p.resolveCompletionBufferSize()) + frame.SetCompletions(completions) + + for { + var out ref.Val + var det *EvalDetails + + if p.observable != nil { + det = &EvalDetails{} + out = p.observable.ObserveExec(frame, func(observed any) { + switch o := observed.(type) { + case interpreter.EvalState: + det.state = o + case *interpreter.CostTracker: + det.costTracker = o + } + }) + } else { + out = p.interpretable.Exec(frame) + } -// Parent implements the Activation interface -func (a *evalActivation) Parent() Activation { - return nil -} + // Communicate errors quickly. + if types.IsError(out) { + var err error = out.(*types.Err) + if errors.Is(err, interpreter.InterruptError{}) { + err = fmt.Errorf("%w: %w", err, context.Cause(ctx)) + } + resCh <- EvalResult{Val: out, EvalDetails: det, Err: err} + return + } -func newEvalActivationPool() *evalActivationPool { - return &evalActivationPool{ - Pool: sync.Pool{ - New: func() any { - return &evalActivation{lazyVars: make(map[string]any)} - }, - }, - } -} + // A concrete (non-unknown) result is final. + unk, isUnknown := out.(*types.Unknown) + if !isUnknown || !unk.HasUnknownFunction() { + resCh <- EvalResult{Val: out, EvalDetails: det, Err: nil} + return + } -type evalActivationPool struct { - sync.Pool -} + // Post-execution dispatch: launch only the async calls required by the unknown result. + frame.DispatchPendingAsyncCalls(unk.IDs()) -// Setup initializes a pooled Activation object with the map input. -func (p *evalActivationPool) Setup(vars map[string]any) *evalActivation { - a := p.Pool.Get().(*evalActivation) - a.vars = vars - return a -} + // The result depends on one or more unresolved async calls. Wait for completions and + // re-evaluate according to the configured drain strategy. + var batch []async.Call -func (p *evalActivationPool) Put(value any) { - a := value.(*evalActivation) - for k := range a.lazyVars { - delete(a.lazyVars, k) - } - p.Pool.Put(a) -} + // Wait for at least one completion (or cancellation). + select { + case id := <-completions: + if call := frame.AsyncCall(id); call != nil { + batch = append(batch, call) + } + case <-ctx.Done(): + resCh <- EvalResult{Val: out, EvalDetails: det, Err: ctx.Err()} + return + } + + // Accumulate completions and consult the strategy. + var timer *time.Timer + reevaluate := false + for !reevaluate { + active := frame.ActiveAsyncCalls() + action := p.drainStrategy.NextAction(batch, active) + if action.Reevaluate { + break + } + + var timeoutCh <-chan time.Time + if action.WaitDuration > 0 { + if timer == nil { + timer = time.NewTimer(action.WaitDuration) + } else { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(action.WaitDuration) + } + timeoutCh = timer.C + } + + select { + case id := <-completions: + if call := frame.AsyncCall(id); call != nil { + batch = append(batch, call) + } + case <-timeoutCh: + reevaluate = true + case <-ctx.Done(): + if timer != nil { + timer.Stop() + } + resCh <- EvalResult{Val: out, EvalDetails: det, Err: ctx.Err()} + return + } + } + if timer != nil { + timer.Stop() + } + } + }() -var ( - // activationPool is an internally managed pool of Activation values that wrap map[string]any inputs - activationPool = newEvalActivationPool() + return resCh +} - // ctxActivationPool is an internally managed pool of Activation values that expose a special #interrupted variable - ctxActivationPool = newCtxEvalActivationPool() -) +// errAsyncRequiresConcurrentEval is returned by the synchronous entry points (Eval, ContextEval) +// when the expression contains asynchronous function calls, which only ConcurrentEval can resolve. +var errAsyncRequiresConcurrentEval = errors.New( + "expression contains asynchronous function calls; use ConcurrentEval") diff --git a/vendor/github.com/google/cel-go/cel/prompt.go b/vendor/github.com/google/cel-go/cel/prompt.go index 1529680fd..f59934827 100644 --- a/vendor/github.com/google/cel-go/cel/prompt.go +++ b/vendor/github.com/google/cel-go/cel/prompt.go @@ -108,7 +108,7 @@ type Prompt struct { // tmpl is the text template base-configuration for rendering text. tmpl *template.Template - // fieldPaths is a flag to enable including reachable field paths in the prompt. + // fieldPaths is a flag to include reachable field paths in the prompt. fieldPaths bool // env reference used to collect variables, functions, and macros available to the prompt. @@ -131,6 +131,12 @@ type promptInst struct { // Render renders the user prompt with the associated context from the prompt template // for use with LLM generators. +// +// User-supplied input is passed as template data via the UserPrompt field, which +// Go's text/template renders as a literal string value. Template action delimiters +// such as {{.Persona}} in the user prompt are never evaluated as template directives +// because text/template only executes directives present in the template definition +// itself, not in data values interpolated at render time. func (p *Prompt) Render(userPrompt string) string { var buffer strings.Builder vars := make([]*promptVariable, len(p.env.Variables())) @@ -178,7 +184,8 @@ func (p *Prompt) Render(userPrompt string) string { Variables: vars, Macros: macs, Functions: funcs, - UserPrompt: userPrompt} + UserPrompt: userPrompt, + } p.tmpl.Execute(&buffer, inst) return buffer.String() } diff --git a/vendor/github.com/google/cel-go/cel/validator.go b/vendor/github.com/google/cel-go/cel/validator.go index 952f88f41..cb7f4c29e 100644 --- a/vendor/github.com/google/cel-go/cel/validator.go +++ b/vendor/github.com/google/cel-go/cel/validator.go @@ -15,6 +15,7 @@ package cel import ( + "context" "fmt" "reflect" "regexp" @@ -25,11 +26,12 @@ import ( ) const ( - durationValidatorName = "cel.validator.duration" - regexValidatorName = "cel.validator.matches" - timestampValidatorName = "cel.validator.timestamp" - homogeneousValidatorName = "cel.validator.homogeneous_literals" - nestingLimitValidatorName = "cel.validator.comprehension_nesting_limit" + durationValidatorName = "cel.validator.duration" + regexValidatorName = "cel.validator.matches" + timestampValidatorName = "cel.validator.timestamp" + homogeneousValidatorName = "cel.validator.homogeneous_literals" + nestingLimitValidatorName = "cel.validator.comprehension_nesting_limit" + bindNestingLimitValidatorName = "cel.validator.bind_nesting_limit" // HomogeneousAggregateLiteralExemptFunctions is the ValidatorConfig key used to configure // the set of function names which are exempt from homogeneous type checks. The expected type @@ -60,6 +62,23 @@ var ( } return nil, fmt.Errorf("invalid validator: %s missing limit", nestingLimitValidatorName) }, + bindNestingLimitValidatorName: func(val *env.Validator) (ASTValidator, error) { + if limit, found := val.ConfigValue("limit"); found { + // In case of protos, config value is of type by google.protobuf.Value, which numeric values are always a double. + if val, isDouble := limit.(float64); isDouble { + if val != float64(int64(val)) { + return nil, fmt.Errorf("invalid validator: %s, limit value is not a whole number: %v", bindNestingLimitValidatorName, limit) + } + return ValidateBindNestingLimit(int(val)), nil + } + + if val, isInt := limit.(int); isInt { + return ValidateBindNestingLimit(val), nil + } + return nil, fmt.Errorf("invalid validator: %s unsupported limit type: %v", bindNestingLimitValidatorName, limit) + } + return nil, fmt.Errorf("invalid validator: %s missing limit", bindNestingLimitValidatorName) + }, durationValidatorName: func(*env.Validator) (ASTValidator, error) { return ValidateDurationLiterals(), nil }, @@ -80,12 +99,20 @@ type ASTValidatorFactory func(*env.Validator) (ASTValidator, error) // ASTValidators configures a set of ASTValidator instances into the target environment. // -// Validators are applied in the order in which the are specified and are treated as singletons. -// The same ASTValidator with a given name will not be applied more than once. +// Validators are applied in the order in which they are specified. +// If an ASTValidator with the same name is already configured, it will be replaced. func ASTValidators(validators ...ASTValidator) EnvOption { return func(e *Env) (*Env, error) { for _, v := range validators { - if !e.HasValidator(v.Name()) { + found := false + for i, existing := range e.validators { + if existing.Name() == v.Name() { + e.validators[i] = v + found = true + break + } + } + if !found { e.validators = append(e.validators, v) } } @@ -232,6 +259,13 @@ func ValidateComprehensionNestingLimit(limit int) ASTValidator { return nestingLimitValidator{limit: limit} } +// ValidateBindNestingLimit ensures that cel.bind() macro nesting does not exceed the specified limit. +// +// This validator can be useful for preventing arbitrarily nested cel.bind() macro calls. +func ValidateBindNestingLimit(limit int) ASTValidator { + return bindNestingLimitValidator{limit: limit} +} + type argChecker func(env *Env, call, arg ast.Expr) error func newFormatValidator(funcName string, argNum int, check argChecker) formatValidator { @@ -284,8 +318,9 @@ func evalCall(env *Env, call, arg ast.Expr) error { if err != nil { return err } - _, _, err = prg.Eval(NoVars()) - return err + resCh := prg.ConcurrentEval(context.Background(), NoVars()) + res := <-resCh + return res.Err } func compileRegex(_ *Env, _, arg ast.Expr) error { @@ -430,8 +465,7 @@ func (v nestingLimitValidator) Validate(e *Env, _ ValidatorConfig, a *ast.AST, i } // When the comprehension has an empty range, continue to the next ancestor // as this comprehension does not have any associated cost. - iterRange := e.AsComprehension().IterRange() - if iterRange.Kind() == ast.ListKind && iterRange.AsList().Size() == 0 { + if isEmptyRangeComprehension(e) { e, hasParent = e.Parent() continue } @@ -445,3 +479,68 @@ func (v nestingLimitValidator) Validate(e *Env, _ ValidatorConfig, a *ast.AST, i } } } + +type bindNestingLimitValidator struct { + limit int +} + +// Name returns the name of the cel.bind nesting limit validator. +func (v bindNestingLimitValidator) Name() string { + return bindNestingLimitValidatorName +} + +// ToConfig converts the ASTValidator to an env.Validator specifying the validator name and the nesting limit +// as an integer value: {"limit": int} +func (v bindNestingLimitValidator) ToConfig() *env.Validator { + return env.NewValidator(v.Name()).SetConfig(map[string]any{"limit": v.limit}) +} + +// Validate implements the ASTValidator interface method. +func (v bindNestingLimitValidator) Validate(e *Env, _ ValidatorConfig, a *ast.AST, iss *Issues) { + root := ast.NavigateAST(a) + comprehensions := ast.MatchDescendants(root, ast.KindMatcher(ast.ComprehensionKind)) + var celBinds []ast.NavigableExpr + for _, comp := range comprehensions { + if isCelBind(comp) { + celBinds = append(celBinds, comp) + } + } + if len(celBinds) <= v.limit { + return + } + for _, comp := range celBinds { + count := 0 + e := comp + hasParent := true + for hasParent { + if isCelBind(e) { + count++ + if count > v.limit { + iss.ReportErrorAtID(comp.ID(), "cel.bind exceeds nesting limit") + break + } + } + e, hasParent = e.Parent() + } + } +} + +func isEmptyRangeComprehension(e ast.NavigableExpr) bool { + if e.Kind() != ast.ComprehensionKind { + return false + } + iterRange := e.AsComprehension().IterRange() + return iterRange.Kind() == ast.ListKind && iterRange.AsList().Size() == 0 +} + +func isCelBind(e ast.NavigableExpr) bool { + if !isEmptyRangeComprehension(e) { + return false + } + compre := e.AsComprehension() + loopCond := compre.LoopCondition() + loopStep := compre.LoopStep() + return compre.IterVar() == unusedIterVar && + loopCond.Kind() == ast.LiteralKind && loopCond.AsLiteral().Value() == false && + loopStep.Kind() == ast.IdentKind && loopStep.AsIdent() == compre.AccuVar() +} diff --git a/vendor/github.com/google/cel-go/checker/cost.go b/vendor/github.com/google/cel-go/checker/cost.go index 5bc6318ed..3d7dd7ec4 100644 --- a/vendor/github.com/google/cel-go/checker/cost.go +++ b/vendor/github.com/google/cel-go/checker/cost.go @@ -159,6 +159,11 @@ func (se SizeEstimate) Union(size SizeEstimate) SizeEstimate { return result } +// AsCost converts a size estimates to an equivalent cost estimate. +func (se SizeEstimate) AsCost() CostEstimate { + return se.MultiplyByCostFactor(1) +} + // CostEstimate represents an estimated cost range and provides add and multiply operations // that do not overflow. type CostEstimate struct { @@ -786,18 +791,26 @@ func (c *coster) functionCost(e ast.Expr, function, overloadID string, target *A return CallEstimate{CostEstimate: c.sizeOrUnknown(args[1]).MultiplyByCostFactor(1).Add(argCostSum())} } // O(nm) functions - case overloads.MatchesString: + case overloads.Matches, overloads.MatchesString: // https://swtch.com/~rsc/regexp/regexp1.html applies to RE2 implementation supported by CEL - if target != nil && len(args) == 1 { + var strNode, regexNode AstNode + if overloadID == overloads.MatchesString && target != nil && len(args) == 1 { + strNode = *target + regexNode = args[0] + } else if overloadID == overloads.Matches && target == nil && len(args) == 2 { + strNode = args[0] + regexNode = args[1] + } + if strNode != nil && regexNode != nil { // Add one to string length for purposes of cost calculation to prevent product of string and regex to be 0 // in case where string is empty but regex is still expensive. - strCost := c.sizeOrUnknown(*target).Add(SizeEstimate{Min: 1, Max: 1}).MultiplyByCostFactor(common.StringTraversalCostFactor) + strCost := c.sizeOrUnknown(strNode).Add(SizeEstimate{Min: 1, Max: 1}).MultiplyByCostFactor(common.StringTraversalCostFactor) // We don't know how many expressions are in the regex, just the string length (a huge // improvement here would be to somehow get a count the number of expressions in the regex or // how many states are in the regex state machine and use that to measure regex cost). // For now, we're making a guess that each expression in a regex is typically at least 4 chars // in length. - regexCost := c.sizeOrUnknown(args[0]).MultiplyByCostFactor(common.RegexStringLengthCostFactor) + regexCost := c.sizeOrUnknown(regexNode).MultiplyByCostFactor(common.RegexStringLengthCostFactor) return CallEstimate{CostEstimate: strCost.Multiply(regexCost).Add(argCostSum())} } case overloads.ContainsString: diff --git a/vendor/github.com/google/cel-go/common/ast/ast.go b/vendor/github.com/google/cel-go/common/ast/ast.go index 3ae2e1063..c8f8f8a02 100644 --- a/vendor/github.com/google/cel-go/common/ast/ast.go +++ b/vendor/github.com/google/cel-go/common/ast/ast.go @@ -172,6 +172,14 @@ func (a *AST) IDs() map[int64]bool { return visitor } +// NodeCount returns the total number of expression nodes in the AST, including macro calls. +func NodeCount(a *AST) int { + if a == nil { + return 0 + } + return len(a.IDs()) +} + // ClearUnusedIDs removes IDs not used in the AST or macro calls from SourceInfo. func (a *AST) ClearUnusedIDs() { ids := a.IDs() diff --git a/vendor/github.com/google/cel-go/common/ast/navigable.go b/vendor/github.com/google/cel-go/common/ast/navigable.go index 13e5777b5..364edfa3a 100644 --- a/vendor/github.com/google/cel-go/common/ast/navigable.go +++ b/vendor/github.com/google/cel-go/common/ast/navigable.go @@ -181,6 +181,29 @@ func PreOrderVisit(expr Expr, visitor Visitor) { visit(expr, visitor, preOrder, 0, 0) } +// ExceedsDepth determines whether the AST contains expressions nested deeper than the specified +// maxDepth. The root expression has depth 0, so a maxDepth of 250 permits expressions nested up +// to and including 250 levels deep. +// +// The traversal is bounded: it descends at most maxDepth+1 levels, so it remains safe to call on +// adversarially deep inputs that could otherwise exhaust the Go stack during later checking or +// planning. A non-positive maxDepth disables the check and returns false. +func ExceedsDepth(a *AST, maxDepth int) bool { + if a == nil || maxDepth <= 0 { + return false + } + exceedsDepth := false + visitor := NewExprVisitor(func(e Expr) { + if nav, ok := e.(NavigableExpr); ok && nav.Depth() >= maxDepth { + exceedsDepth = true + } + }) + // Bound the walk to maxDepth+1 levels so it never recurses past the first level that exceeds + // the limit, keeping the check itself safe on the deep inputs it guards against. + visit(NavigateAST(a), visitor, postOrder, 0, maxDepth+1) + return exceedsDepth +} + type visitOrder int const ( diff --git a/vendor/github.com/google/cel-go/common/containers/container.go b/vendor/github.com/google/cel-go/common/containers/container.go index fc146b6fc..fcfcdfc3f 100644 --- a/vendor/github.com/google/cel-go/common/containers/container.go +++ b/vendor/github.com/google/cel-go/common/containers/container.go @@ -227,7 +227,7 @@ func Abbrevs(qualifiedNames ...string) ContainerOption { } alias := qn[ind+1:] var err error - c, err = aliasAs("abbreviation", qn, alias)(c) + c, err = aliasAs("abbreviation", qn, alias, true)(c) if err != nil { return nil, err } @@ -236,31 +236,32 @@ func Abbrevs(qualifiedNames ...string) ContainerOption { } } -// Alias associates a fully-qualified name with a user-defined alias. +// Alias associates a name with a user-defined alias. // // In general, Abbrevs is preferred to Alias since the names generated from the Abbrevs option // are more easily traced back to source code. The Alias option is useful for propagating alias // configuration from one Container instance to another, and may also be useful for remapping // poorly chosen protobuf message / package names. -// -// Note: all of the rules that apply to Abbrevs also apply to Alias. func Alias(qualifiedName, alias string) ContainerOption { - return aliasAs("alias", qualifiedName, alias) + return aliasAs("alias", qualifiedName, alias, false) } -func aliasAs(kind, qualifiedName, alias string) ContainerOption { +func aliasAs(kind, qualifiedName, alias string, requireQualified bool) ContainerOption { return func(c *Container) (*Container, error) { if len(alias) == 0 || strings.Contains(alias, ".") { return nil, fmt.Errorf( "%s must be non-empty and simple (not qualified): %s=%s", kind, kind, alias) } + if len(qualifiedName) == 0 { + return nil, fmt.Errorf("%s must refer to a valid name: %s", kind, qualifiedName) + } if qualifiedName[0:1] == "." { return nil, fmt.Errorf("qualified name must not begin with a leading '.': %s", qualifiedName) } ind := strings.LastIndex(qualifiedName, ".") - if ind <= 0 || ind == len(qualifiedName)-1 { + if ind == len(qualifiedName)-1 || (requireQualified && ind <= 0) { return nil, fmt.Errorf("%s must refer to a valid qualified name: %s", kind, qualifiedName) } diff --git a/vendor/github.com/google/cel-go/common/decls/decls.go b/vendor/github.com/google/cel-go/common/decls/decls.go index cd4d3a565..51cb689e5 100644 --- a/vendor/github.com/google/cel-go/common/decls/decls.go +++ b/vendor/github.com/google/cel-go/common/decls/decls.go @@ -16,6 +16,7 @@ package decls import ( + "context" "fmt" "strings" @@ -316,8 +317,12 @@ func (f *FunctionDecl) HasLateBinding() bool { if f == nil { return false } + if f.singleton != nil && f.singleton.Async != nil { + return true + } for _, oID := range f.overloadOrdinals { - if f.overloads[oID].HasLateBinding() { + o := f.overloads[oID] + if o.HasLateBinding() { return true } } @@ -342,6 +347,7 @@ func (f *FunctionDecl) Bindings() ([]*functions.Overload, error) { Unary: o.guardedUnaryOp(f.Name(), f.disableTypeGuards), Binary: o.guardedBinaryOp(f.Name(), f.disableTypeGuards), Function: o.guardedFunctionOp(f.Name(), f.disableTypeGuards), + Async: o.guardedAsyncOp(f.Name(), f.disableTypeGuards), OperandTrait: o.OperandTrait(), NonStrict: o.IsNonStrict(), } @@ -362,6 +368,7 @@ func (f *FunctionDecl) Bindings() ([]*functions.Overload, error) { Unary: f.singleton.Unary, Binary: f.singleton.Binary, Function: f.singleton.Function, + Async: f.singleton.Async, OperandTrait: f.singleton.OperandTrait, }, } @@ -380,6 +387,7 @@ func (f *FunctionDecl) Bindings() ([]*functions.Overload, error) { Unary: overloads[0].Unary, Binary: overloads[0].Binary, Function: overloads[0].Function, + Async: overloads[0].Async, NonStrict: overloads[0].NonStrict, OperandTrait: overloads[0].OperandTrait, }), nil @@ -538,6 +546,30 @@ func SingletonFunctionBinding(fn functions.FunctionOp, traits ...int) FunctionOp } } +// SingletonAsyncBinding creates a singleton async function definition to be used with all function overloads. +// The provided function is called in its own goroutine with the provided context. The function should +// block until the result is available, and the framework manages goroutine and channel lifecycle. +// +// Note, this approach works well if operand is expected to have a specific trait which it implements, +// e.g. traits.ContainerType. Otherwise, prefer per-overload async bindings. +func SingletonAsyncBinding(fn functions.BlockingAsyncOp, traits ...int) FunctionOpt { + trait := 0 + for _, t := range traits { + trait = trait | t + } + return func(f *FunctionDecl) (*FunctionDecl, error) { + if f.singleton != nil { + return nil, fmt.Errorf("function already has a singleton binding: %s", f.Name()) + } + f.singleton = &functions.Overload{ + Operator: f.Name(), + Async: wrapAsyncOp(fn), + OperandTrait: trait, + } + return f, nil + } +} + // Overload defines a new global overload with an overload id, argument types, and result type. Through the // use of OverloadOpt options, the overload may also be configured with a binding, an operand trait, and to // be non-strict. @@ -622,6 +654,8 @@ type OverloadDecl struct { binaryOp functions.BinaryOp // functionOp is a catch-all for zero-arity and three-plus arity functions. functionOp functions.FunctionOp + // asyncOp is an asynchronous function binding that returns a channel. + asyncOp functions.AsyncOp } // Examples returns a list of string examples for the overload. @@ -677,7 +711,7 @@ func (o *OverloadDecl) HasLateBinding() bool { if o == nil { return false } - return o.hasLateBinding + return o.hasLateBinding || o.asyncOp != nil } // OperandTrait returns the trait mask of the first operand to the overload call, e.g. @@ -750,7 +784,7 @@ func (o *OverloadDecl) SignatureOverlaps(other *OverloadDecl) bool { // HasBinding indicates whether the overload already has a definition. func (o *OverloadDecl) HasBinding() bool { - return o != nil && (o.unaryOp != nil || o.binaryOp != nil || o.functionOp != nil) + return o != nil && (o.unaryOp != nil || o.binaryOp != nil || o.functionOp != nil || o.asyncOp != nil) } // guardedUnaryOp creates an invocation guard around the provided unary operator, if one is defined. @@ -792,6 +826,22 @@ func (o *OverloadDecl) guardedFunctionOp(funcName string, disableTypeGuards bool } } +// guardedAsyncOp creates an invocation guard around the provided async function binding, if one is provided. +func (o *OverloadDecl) guardedAsyncOp(funcName string, disableTypeGuards bool) functions.AsyncOp { + if o.asyncOp == nil { + return nil + } + return func(ctx context.Context, args ...ref.Val) <-chan ref.Val { + if !o.matchesRuntimeSignature(disableTypeGuards, args...) { + ch := make(chan ref.Val, 1) + ch <- MaybeNoSuchOverload(funcName, args...) + close(ch) + return ch + } + return o.asyncOp(ctx, args...) + } +} + // matchesRuntimeUnarySignature indicates whether the argument type is runtime assiganble to the overload's expected argument. func (o *OverloadDecl) matchesRuntimeUnarySignature(disableTypeGuards bool, arg ref.Val) bool { return matchRuntimeArgType(o.IsNonStrict(), disableTypeGuards, o.ArgTypes()[0], arg) && @@ -825,6 +875,8 @@ func matchRuntimeArgType(nonStrict, disableTypeGuards bool, argType *types.Type, if nonStrict && (disableTypeGuards || types.IsUnknownOrError(arg)) { return true } + // Note, early returns and unknown aggregation happen in the interpretable.go file; however, this check is here + // for defense in depth or for scenarios where someone manipulates bindings to offer their own dispatch logic. if types.IsUnknownOrError(arg) { return false } @@ -897,6 +949,40 @@ func FunctionBinding(binding functions.FunctionOp) OverloadOpt { } } +// AsyncBinding provides the implementation of an asynchronous overload. The provided function +// is called in its own goroutine with the provided context. The function should block until +// the result is available, and the framework manages goroutine and channel lifecycle. +// +// This follows the same pattern used by gRPC-Go and other major Go frameworks where user +// code is synchronous and the framework manages concurrency. +func AsyncBinding(fn functions.BlockingAsyncOp) OverloadOpt { + return func(o *OverloadDecl) (*OverloadDecl, error) { + if o.HasBinding() { + return nil, fmt.Errorf("overload already has a binding: %s", o.ID()) + } + if o.hasLateBinding { + return nil, fmt.Errorf("overload already has a late binding: %s", o.ID()) + } + o.asyncOp = wrapAsyncOp(fn) + return o, nil + } +} + +// wrapAsyncOp adapts a blocking function into the channel-based AsyncOp used internally. +// +// The blocking function is invoked synchronously and its result delivered on a buffered channel. +// The interpreter always invokes an AsyncOp from a dedicated goroutine, so running the blocking +// call inline here keeps the framework to a single goroutine per async call rather than spawning +// an additional one to bridge blocking-to-channel. +func wrapAsyncOp(fn functions.BlockingAsyncOp) functions.AsyncOp { + return func(ctx context.Context, args ...ref.Val) <-chan ref.Val { + ch := make(chan ref.Val, 1) + ch <- fn(ctx, args...) + close(ch) + return ch + } +} + // LateFunctionBinding indicates that the function has a binding which is not known at compile time. // This is useful for functions which have side-effects or are not deterministically computable. func LateFunctionBinding() OverloadOpt { diff --git a/vendor/github.com/google/cel-go/common/env/BUILD.bazel b/vendor/github.com/google/cel-go/common/env/BUILD.bazel index b2e0c2931..261da924d 100644 --- a/vendor/github.com/google/cel-go/common/env/BUILD.bazel +++ b/vendor/github.com/google/cel-go/common/env/BUILD.bazel @@ -23,12 +23,14 @@ go_library( name = "go_default_library", srcs = [ "env.go", + "io.go", ], importpath = "github.com/google/cel-go/common/env", deps = [ "//common:go_default_library", "//common/decls:go_default_library", "//common/types:go_default_library", + "@in_yaml_go_yaml_v3//:go_default_library", ], ) @@ -37,6 +39,7 @@ go_test( size = "small", srcs = [ "env_test.go", + "io_test.go", ], data = glob(["testdata/**"]), embed = [":go_default_library"], @@ -45,6 +48,7 @@ go_test( "//common/operators:go_default_library", "//common/overloads:go_default_library", "//common/types:go_default_library", + "@com_github_google_go_cmp//cmp:go_default_library", "@in_yaml_go_yaml_v3//:go_default_library", ], ) diff --git a/vendor/github.com/google/cel-go/common/env/env.go b/vendor/github.com/google/cel-go/common/env/env.go index 85ec85cd0..936036ed2 100644 --- a/vendor/github.com/google/cel-go/common/env/env.go +++ b/vendor/github.com/google/cel-go/common/env/env.go @@ -258,7 +258,9 @@ type Variable struct { // Type represents the type declaration for the variable. // - // Deprecated: use the embedded *TypeDesc fields directly. + // When serialized, 'type' is used for shorthand specifier string. + // + // Use GetType() for getting the effective type. Type *TypeDesc `yaml:"type,omitempty"` // TypeDesc is an embedded set of fields allowing for the specification of the Variable type. @@ -276,6 +278,9 @@ func (v *Variable) Validate() error { if err := v.GetType().Validate(); err != nil { return fmt.Errorf("invalid variable %q: %w", v.Name, err) } + if v.GetType().IsTypeParam { + return fmt.Errorf("invalid variable %q: variables cannot be type parameters", v.Name) + } return nil } @@ -844,6 +849,34 @@ func (td *TypeDesc) Validate() error { return nil } +func formatSpecifierImpl(td *TypeDesc, sb *strings.Builder) { + if td.IsTypeParam { + sb.WriteRune('~') + sb.WriteString(td.TypeName) + return + } + sb.WriteString(td.TypeName) + l := len(td.Params) + if l < 1 { + return + } + sb.WriteRune('<') + for i, p := range td.Params { + formatSpecifierImpl(p, sb) + if i < l-1 { + sb.WriteString(", ") + } + } + sb.WriteRune('>') +} + +// SpecifierFormat returns the short text representation of the type. e.g. "map" +func (td *TypeDesc) SpecifierFormat() string { + var sb strings.Builder + formatSpecifierImpl(td, &sb) + return sb.String() +} + // AsCELType converts the serializable object to a *types.Type value. func (td *TypeDesc) AsCELType(tp types.Provider) (*types.Type, error) { err := td.Validate() @@ -853,6 +886,27 @@ func (td *TypeDesc) AsCELType(tp types.Provider) (*types.Type, error) { switch td.TypeName { case "dyn": return types.DynType, nil + // short aliases for WKTs + case "duration": + return types.DurationType, nil + case "timestamp": + return types.TimestampType, nil + case "any": + return types.AnyType, nil + case "null", "null_type": + return types.NullType, nil + case "bool_wrapper": + return types.NewNullableType(types.BoolType), nil + case "bytes_wrapper": + return types.NewNullableType(types.BytesType), nil + case "double_wrapper": + return types.NewNullableType(types.DoubleType), nil + case "int_wrapper": + return types.NewNullableType(types.IntType), nil + case "uint_wrapper": + return types.NewNullableType(types.UintType), nil + case "string_wrapper": + return types.NewNullableType(types.StringType), nil case "map": kt, err := td.Params[0].AsCELType(tp) if err != nil { @@ -926,6 +980,15 @@ func SerializeTypeDesc(t *types.Type) *TypeDesc { for _, p := range t.Parameters() { params = append(params, SerializeTypeDesc(p)) } + // Special types, these aren't useful for describing environments. + switch t.Kind() { + case types.ErrorKind: + typeName = "*error*" + case types.UnknownKind: + typeName = "*unknown*" + case types.UnspecifiedKind: + typeName = "*unspecified type*" + } return NewTypeDesc(typeName, params...) } diff --git a/vendor/github.com/google/cel-go/common/env/io.go b/vendor/github.com/google/cel-go/common/env/io.go new file mode 100644 index 000000000..ec126f9ce --- /dev/null +++ b/vendor/github.com/google/cel-go/common/env/io.go @@ -0,0 +1,271 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package env + +import ( + "errors" + "fmt" + + "go.yaml.in/yaml/v3" +) + +type internalTypeDesc struct { + TypeName string `yaml:"type_name"` + Params []*TypeDesc `yaml:"params,omitempty"` + IsTypeParam bool `yaml:"is_type_param,omitempty"` +} + +// Embedding TypeDesc in variable causes issues with customizing +// unmarshalling / marshalling. Work around with a parallel type. +type internalVariable struct { + Name string `yaml:"name"` + Description string `yaml:"description,omitempty"` + + // Type represents the type declaration for the variable. + Type *TypeDesc `yaml:"type,omitempty"` + + TypeName string `yaml:"type_name"` + Params []*TypeDesc `yaml:"params,omitempty"` + IsTypeParam bool `yaml:"is_type_param,omitempty"` +} + +// UnmarshalYAML implements yaml.Unmarshal +func (v *Variable) UnmarshalYAML(n *yaml.Node) error { + buf := internalVariable{} + err := n.Decode(&buf) + if err != nil { + return err + } + v.Name = buf.Name + v.Description = buf.Description + if buf.TypeName != "" { + v.TypeDesc = &TypeDesc{ + TypeName: buf.TypeName, + Params: buf.Params, + IsTypeParam: buf.IsTypeParam, + } + } else if buf.Type != nil { + v.TypeDesc = buf.Type + } + return nil +} + +// MarshalYAML implements yaml.Marshaler +func (v *Variable) MarshalYAML() (any, error) { + // The presence of an unmarshaller alters the default marshaller behavior so + // provide a simple marshal implementation. + buf := internalVariable{ + Name: v.Name, + Description: v.Description, + } + if t := v.GetType(); t != nil { + buf.TypeName = t.TypeName + buf.Params = t.Params + buf.IsTypeParam = t.IsTypeParam + } + return &buf, nil +} + +// UnmarshalYAML implements yaml.Unmarshaler +func (td *TypeDesc) UnmarshalYAML(n *yaml.Node) error { + if td == nil { + return fmt.Errorf("unexpected Unmarshal for TypeDesc at: %d", n.Line) + } + if n.Kind == yaml.ScalarNode { + o, err := ParseTypeDesc(n.Value) + if err != nil { + return err + } + *td = *o + return nil + } + + if n.Kind != yaml.MappingNode { + return errors.New("unsupported yaml for TypeDesc") + } + + buf := internalTypeDesc{} + err := n.Decode(&buf) + if err != nil { + return err + } + td.TypeName = buf.TypeName + td.Params = buf.Params + td.IsTypeParam = buf.IsTypeParam + return nil +} + +type typeDescParser struct { + text string + pos int + length int +} + +// ParseTypeDesc parses a TypeDesc from the type specifier format: "map" +func ParseTypeDesc(text string) (*TypeDesc, error) { + p := &typeDescParser{text: text, length: len(text)} + res, err := p.parseTypeElem() + if err != nil { + return nil, fmt.Errorf("failed to parse type %q: %v", text, err) + } + p.skipWhitespace() + if p.pos < p.length { + return nil, fmt.Errorf("unexpected character %q at position %d in %q", p.text[p.pos], p.pos, text) + } + return res, nil +} + +func (p *typeDescParser) parseConcreteType() (*TypeDesc, error) { + id, err := p.parseNamespaceIdentifier() + if err != nil { + return nil, err + } + if p.pos < p.length && p.text[p.pos] == '<' { + p.pos++ // consume '<' + var params []*TypeDesc + for { + p.skipWhitespace() + param, err := p.parseTypeElem() + if err != nil { + return nil, err + } + params = append(params, param) + p.skipWhitespace() + if p.pos < p.length && p.text[p.pos] == ',' { + p.pos++ // consume ',' + continue + } + if p.pos < p.length && p.text[p.pos] == '>' { + p.pos++ // consume '>' + break + } + return nil, fmt.Errorf("expected ',' or '>' at position %d", p.pos) + } + return NewTypeDesc(id, params...), nil + } + return NewTypeDesc(id), nil +} + +func (p *typeDescParser) parseTypeElem() (*TypeDesc, error) { + p.skipWhitespace() + if p.pos < p.length && p.text[p.pos] == '~' { + p.pos++ // consume '~' + id, err := p.parseTypeParamIdent() + if err != nil { + return nil, err + } + return NewTypeParam(id), nil + } + return p.parseConcreteType() +} + +func (p *typeDescParser) parseNamespaceIdentifier() (string, error) { + p.skipWhitespace() + var id string + for p.pos < p.length && p.text[p.pos] != '<' { + c := p.text[p.pos] + if c == '.' { + id += "." + p.pos++ // consume '.' + } + ident, err := p.parseIdentifier() + if err != nil { + return "", err + } + id += ident + p.skipWhitespace() + if p.pos < p.length && p.text[p.pos] != '.' { + break + } + } + if id == "" { + return "", fmt.Errorf("missing identifier at position %d", p.pos) + } + return id, nil +} + +func (p *typeDescParser) parseIdentifier() (string, error) { + p.skipWhitespace() + if p.pos >= p.length { + return "", fmt.Errorf("unexpected end of input") + } + start := p.pos + c := p.text[p.pos] + if !isAlpha(c) && c != '_' { + return "", fmt.Errorf("identifier is expected, but %q was found at position %d", c, p.pos) + } + p.pos++ + for p.pos < p.length { + c := p.text[p.pos] + if !isAlphaNumeric(c) && c != '_' { + break + } + p.pos++ + } + return p.text[start:p.pos], nil +} + +func (p *typeDescParser) parseTypeParamIdent() (string, error) { + p.skipWhitespace() + if p.pos >= p.length { + return "", fmt.Errorf("unexpected end of input") + } + c := p.text[p.pos] + if !isAlpha(c) { + return "", fmt.Errorf("invalid type parameter identifier %q at position %d, must be a single character from A-Z", c, p.pos) + } + p.pos++ + if p.pos < p.length && isAlpha(p.text[p.pos]) { + return "", fmt.Errorf("invalid type param, must have a single alphabetic character at position %d", p.pos) + } + return string(c), nil +} + +func (p *typeDescParser) skipWhitespace() { + for p.pos < p.length && p.text[p.pos] == ' ' { + p.pos++ + } +} + +func isAlpha(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +func isAlphaNumeric(c byte) bool { + return isAlpha(c) || (c >= '0' && c <= '9') +} + +// ConfigFromYAML returns a config from YAML source. +// +// Adds custom parsing logic for normalizing shorthand for specifiying some fields +// in a YAML document (mainly the type-specifier shorthand). +// +// Using yaml.Unmarshal with any implementation should be sufficient for most +// cases. +func ConfigFromYAML(data []byte) (*Config, error) { + c := &Config{} + e := yaml.Unmarshal(data, c) + if e != nil { + return nil, e + } + return c, nil +} + +// ConfigToYAML returns the config serialized to YAML +// +// Provided as a convenience wrapper around a tested YAML Marshaler. +func ConfigToYAML(c *Config) ([]byte, error) { + return yaml.Marshal(c) +} diff --git a/vendor/github.com/google/cel-go/common/functions/functions.go b/vendor/github.com/google/cel-go/common/functions/functions.go index 67f4a5944..0c00781d9 100644 --- a/vendor/github.com/google/cel-go/common/functions/functions.go +++ b/vendor/github.com/google/cel-go/common/functions/functions.go @@ -15,7 +15,11 @@ // Package functions defines the standard builtin functions supported by the interpreter package functions -import "github.com/google/cel-go/common/types/ref" +import ( + "context" + + "github.com/google/cel-go/common/types/ref" +) // Overload defines a named overload of a function, indicating an operand trait // which must be present on the first argument to the overload as well as one @@ -41,21 +45,37 @@ type Overload struct { // Binary defines the overload with a BinaryOp implementation. May be nil. Binary BinaryOp - // Function defines the overload with a FunctionOp implementation. May be - // nil. + // Function defines the overload with a FunctionOp implementation. May be nil. Function FunctionOp + // Async defines the overload with an AsyncOp implementation. May be nil. + Async AsyncOp + // NonStrict specifies whether the Overload will tolerate arguments that // are types.Err or types.Unknown. NonStrict bool } // UnaryOp is a function that takes a single value and produces an output. -type UnaryOp func(value ref.Val) ref.Val +type UnaryOp func(ref.Val) ref.Val // BinaryOp is a function that takes two values and produces an output. -type BinaryOp func(lhs ref.Val, rhs ref.Val) ref.Val +type BinaryOp func(ref.Val, ref.Val) ref.Val // FunctionOp is a function with accepts zero or more arguments and produces // a value or error as a result. -type FunctionOp func(values ...ref.Val) ref.Val +type FunctionOp func(...ref.Val) ref.Val + +// AsyncOp is a function that accepts zero or more arguments and produces +// a value or error asynchronously via a channel. +// +// AsyncOp is an internal interface intended for use by CEL to manage goroutines and +// channels associated with async calls. For public API usage, use BlockingAsyncOp. +// Implementers should listen for context cancellation on the provided context for +// resource cleanup. +type AsyncOp func(context.Context, ...ref.Val) <-chan ref.Val + +// BlockingAsyncOp is a function that accepts zero or more arguments and blocks until +// the result is available. When used with AsyncBinding, the framework runs the function +// in its own goroutine and manages channel lifecycle internally. +type BlockingAsyncOp func(context.Context, ...ref.Val) ref.Val diff --git a/vendor/github.com/google/cel-go/common/overloads/overloads.go b/vendor/github.com/google/cel-go/common/overloads/overloads.go index 9d50f4367..0e3a72449 100644 --- a/vendor/github.com/google/cel-go/common/overloads/overloads.go +++ b/vendor/github.com/google/cel-go/common/overloads/overloads.go @@ -291,7 +291,6 @@ const ( const ( DurationToDuration = "duration_to_duration" StringToDuration = "string_to_duration" - IntToDuration = "int64_to_duration" ) // Convert to dyn diff --git a/vendor/github.com/google/cel-go/common/runes/buffer.go b/vendor/github.com/google/cel-go/common/runes/buffer.go index 021198224..58dd33e13 100644 --- a/vendor/github.com/google/cel-go/common/runes/buffer.go +++ b/vendor/github.com/google/cel-go/common/runes/buffer.go @@ -16,6 +16,7 @@ package runes import ( + "fmt" "strings" "unicode/utf8" ) @@ -113,45 +114,64 @@ var _ Buffer = &supplementalBuffer{} var nilBuffer = &emptyBuffer{} +// SizeLimitError indicates that the input exceeded the configured code point limit. +type SizeLimitError struct { + Size int + Limit int +} + +func (e *SizeLimitError) Error() string { + return fmt.Sprintf("expression code point size exceeds limit: size: %d, limit %d", e.Size, e.Limit) +} + // NewBuffer returns an efficient implementation of Buffer for the given text based on the ranges of // the encoded code points contained within. -// -// Code points are represented as an array of byte, uint16, or rune. This approach ensures that -// each index represents a code point by itself without needing to use an array of rune. At first -// we assume all code points are less than or equal to '\u007f'. If this holds true, the -// underlying storage is a byte array containing only ASCII characters. If we encountered a code -// point above this range but less than or equal to '\uffff' we allocate a uint16 array, copy the -// elements of previous byte array to the uint16 array, and continue. If this holds true, the -// underlying storage is a uint16 array containing only Unicode characters in the Basic Multilingual -// Plane. If we encounter a code point above '\uffff' we allocate an rune array, copy the previous -// elements of the byte or uint16 array, and continue. The underlying storage is an rune array -// containing any Unicode character. func NewBuffer(data string) Buffer { - buf, _ := newBuffer(data, false) + buf, _, _ := newBufferWithLimit(data, false, -1) return buf } // NewBufferAndLineOffsets returns an efficient implementation of Buffer for the given text based on // the ranges of the encoded code points contained within, as well as returning the line offsets. -// -// Code points are represented as an array of byte, uint16, or rune. This approach ensures that -// each index represents a code point by itself without needing to use an array of rune. At first -// we assume all code points are less than or equal to '\u007f'. If this holds true, the -// underlying storage is a byte array containing only ASCII characters. If we encountered a code -// point above this range but less than or equal to '\uffff' we allocate a uint16 array, copy the -// elements of previous byte array to the uint16 array, and continue. If this holds true, the -// underlying storage is a uint16 array containing only Unicode characters in the Basic Multilingual -// Plane. If we encounter a code point above '\uffff' we allocate an rune array, copy the previous -// elements of the byte or uint16 array, and continue. The underlying storage is an rune array -// containing any Unicode character. func NewBufferAndLineOffsets(data string) (Buffer, []int32) { - return newBuffer(data, true) + buf, offs, _ := newBufferWithLimit(data, true, -1) + return buf, offs +} + +// NewBufferAndLineOffsetsWithLimit returns an efficient implementation of Buffer for the given text +// and enforces a code point limit while constructing the buffer. +func NewBufferAndLineOffsetsWithLimit(data string, limit int) (Buffer, []int32, error) { + if limit < 0 || len(data) <= limit { + return newBufferWithLimit(data, true, -1) + } + return newBufferWithLimit(data, true, limit) +} + +func countRemainingCodePoints(data string, idx int, count int) int { + for idx < len(data) { + _, s := utf8.DecodeRuneInString(data[idx:]) + idx += s + count++ + } + return count } -func newBuffer(data string, lines bool) (Buffer, []int32) { +func newBufferWithLimit(data string, lines bool, limit int) (Buffer, []int32, error) { if len(data) == 0 { - return nilBuffer, []int32{0} + return nilBuffer, []int32{0}, nil + } + if limit >= 0 && len(data) > limit { + size := countRemainingCodePoints(data, 0, 0) + if size > limit { + return nil, nil, &SizeLimitError{ + Size: size, + Limit: limit, + } + } } + + // The resulting buffers store one element per code point, so the worst case + // element count never exceeds len(data). var ( idx = 0 off int32 = 0 @@ -195,7 +215,8 @@ func newBuffer(data string, lines bool) (Buffer, []int32) { } return &asciiBuffer{ arr: buf8, - }, offs + }, offs, nil + copy16: for idx < len(data) { r, s := utf8.DecodeRuneInString(data[idx:]) @@ -222,7 +243,8 @@ copy16: } return &basicBuffer{ arr: buf16, - }, offs + }, offs, nil + copy32: for idx < len(data) { r, s := utf8.DecodeRuneInString(data[idx:]) @@ -238,5 +260,5 @@ copy32: } return &supplementalBuffer{ arr: buf32, - }, offs + }, offs, nil } diff --git a/vendor/github.com/google/cel-go/common/source.go b/vendor/github.com/google/cel-go/common/source.go index ec79cb545..9187e9b5c 100644 --- a/vendor/github.com/google/cel-go/common/source.go +++ b/vendor/github.com/google/cel-go/common/source.go @@ -74,6 +74,12 @@ func NewTextSource(text string) Source { return NewStringSource(text, "") } +// NewTextSourceWithLimit creates a new Source from the input text string while +// enforcing a maximum code point count when needed. +func NewTextSourceWithLimit(text string, limit int) (Source, error) { + return NewStringSourceWithLimit(text, "", limit) +} + // NewStringSource creates a new Source from the given contents and description. func NewStringSource(contents string, description string) Source { // Compute line offsets up front as they are referred to frequently. @@ -85,6 +91,23 @@ func NewStringSource(contents string, description string) Source { } } +// NewStringSourceWithLimit creates a new Source from the given contents and +// description while enforcing a maximum code point count when needed. +func NewStringSourceWithLimit(contents string, description string, limit int) (Source, error) { + if limit < 0 || len(contents) <= limit { + return NewStringSource(contents, description), nil + } + buf, offs, err := runes.NewBufferAndLineOffsetsWithLimit(contents, limit) + if err != nil { + return nil, err + } + return &sourceImpl{ + Buffer: buf, + description: description, + lineOffsets: offs, + }, nil +} + // NewInfoSource creates a new Source from a SourceInfo. func NewInfoSource(info *exprpb.SourceInfo) Source { return &sourceImpl{ diff --git a/vendor/github.com/google/cel-go/common/stdlib/standard.go b/vendor/github.com/google/cel-go/common/stdlib/standard.go index 4040a4f5c..d2313bef1 100644 --- a/vendor/github.com/google/cel-go/common/stdlib/standard.go +++ b/vendor/github.com/google/cel-go/common/stdlib/standard.go @@ -16,6 +16,7 @@ package stdlib import ( + "math" "strconv" "strings" "time" @@ -310,6 +311,9 @@ func init() { argTypes(types.DurationType, types.DurationType), types.BoolType, decls.OverloadExamples(`duration('1ms') < duration('1s') // true`)), decls.SingletonBinaryBinding(func(lhs, rhs ref.Val) ref.Val { + if isNaN(lhs) || isNaN(rhs) { + return types.False + } cmp := lhs.(traits.Comparer).Compare(rhs) if cmp == types.IntNegOne { return types.True @@ -367,6 +371,9 @@ func init() { argTypes(types.DurationType, types.DurationType), types.BoolType, decls.OverloadExamples(`duration('1ms') <= duration('1s') // true`)), decls.SingletonBinaryBinding(func(lhs, rhs ref.Val) ref.Val { + if isNaN(lhs) || isNaN(rhs) { + return types.False + } cmp := lhs.(traits.Comparer).Compare(rhs) if cmp == types.IntNegOne || cmp == types.IntZero { return types.True @@ -424,6 +431,9 @@ func init() { argTypes(types.DurationType, types.DurationType), types.BoolType, decls.OverloadExamples(`duration('1ms') > duration('1us') // true`)), decls.SingletonBinaryBinding(func(lhs, rhs ref.Val) ref.Val { + if isNaN(lhs) || isNaN(rhs) { + return types.False + } cmp := lhs.(traits.Comparer).Compare(rhs) if cmp == types.IntOne { return types.True @@ -481,6 +491,9 @@ func init() { argTypes(types.DurationType, types.DurationType), types.BoolType, decls.OverloadExamples(`duration('60s') >= duration('1m') // true`)), decls.SingletonBinaryBinding(func(lhs, rhs ref.Val) ref.Val { + if isNaN(lhs) || isNaN(rhs) { + return types.False + } cmp := lhs.(traits.Comparer).Compare(rhs) if cmp == types.IntOne || cmp == types.IntZero { return types.True @@ -605,8 +618,6 @@ func init() { decls.Overload(overloads.DurationToDuration, argTypes(types.DurationType), types.DurationType, decls.OverloadExamples(`duration(duration('1s')) // duration('1s')`), decls.UnaryBinding(identity)), - decls.Overload(overloads.IntToDuration, argTypes(types.IntType), types.DurationType, - decls.UnaryBinding(convertToType(types.DurationType))), decls.Overload(overloads.StringToDuration, argTypes(types.StringType), types.DurationType, decls.OverloadExamples(`duration('1h2m3s') // duration('3723s')`), decls.UnaryBinding(convertToType(types.DurationType)))), @@ -928,6 +939,11 @@ func noBinaryOverrides(rhs, lhs ref.Val) ref.Val { return types.NoSuchOverloadErr() } +func isNaN(val ref.Val) bool { + d, ok := val.(types.Double) + return ok && math.IsNaN(float64(d)) +} + func noFunctionOverrides(args ...ref.Val) ref.Val { return types.NoSuchOverloadErr() } diff --git a/vendor/github.com/google/cel-go/common/types/bytes.go b/vendor/github.com/google/cel-go/common/types/bytes.go index 88da05315..2eefb5d7f 100644 --- a/vendor/github.com/google/cel-go/common/types/bytes.go +++ b/vendor/github.com/google/cel-go/common/types/bytes.go @@ -44,7 +44,10 @@ func (b Bytes) Add(other ref.Val) ref.Val { if !ok { return ValOrErr(other, "no such overload") } - return append(b, otherBytes...) + sum := make([]byte, 0, len(b)+len(otherBytes)) + sum = append(sum, b...) + sum = append(sum, otherBytes...) + return Bytes(sum) } // Compare implements traits.Comparer interface method by lexicographic ordering. diff --git a/vendor/github.com/google/cel-go/common/types/string.go b/vendor/github.com/google/cel-go/common/types/string.go index 5f5a43358..1335903a7 100644 --- a/vendor/github.com/google/cel-go/common/types/string.go +++ b/vendor/github.com/google/cel-go/common/types/string.go @@ -122,7 +122,11 @@ func (s String) ConvertToType(typeVal ref.Type) ref.Val { return durationOf(d) } case TimestampType: - if t, err := time.Parse(time.RFC3339, s.Value().(string)); err == nil { + str := s.Value().(string) + if !isStrictRFC3339(str) { + return NewErr("invalid RFC 3339 timestamp %q", str) + } + if t, err := time.Parse(time.RFC3339, str); err == nil { if t.Unix() < minUnixTime || t.Unix() > maxUnixTime { return celErrTimestampOverflow } diff --git a/vendor/github.com/google/cel-go/common/types/timestamp.go b/vendor/github.com/google/cel-go/common/types/timestamp.go index 060caf6bb..62a020d97 100644 --- a/vendor/github.com/google/cel-go/common/types/timestamp.go +++ b/vendor/github.com/google/cel-go/common/types/timestamp.go @@ -17,9 +17,11 @@ package types import ( "fmt" "reflect" + "regexp" "strconv" "strings" "time" + "unicode" "github.com/google/cel-go/common/overloads" "github.com/google/cel-go/common/types/ref" @@ -52,6 +54,79 @@ const ( maxUnixTime int64 = 253402300799 ) +// strictRFC3339Pattern gates the strings accepted by the `timestamp()` overload. +// time.Parse accepts inputs that RFC 3339 forbids: a ',' fractional-second +// separator, single-digit time fields, and numeric offsets whose hours exceed +// 23 or minutes exceed 59. Those slip past unnoticed and shift the parsed +// instant, so they are rejected before time.Parse runs. Month and day are held +// to the grammar ranges 01-12 and 01-31; the remaining calendar validation +// (day-of-month vs. month, leap years) is left to time.Parse. +// +// isStrictRFC3339 is the implementation used on the conversion path; the pattern +// is retained as the reference the scan is conformance tested against. +var strictRFC3339Pattern = regexp.MustCompile( + `^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])[Tt]([01]\d|2[0-3]):[0-5]\d:([0-5]\d|60)(\.\d+)?([Zz]|[+-]([01]\d|2[0-3]):[0-5]\d)$`) + +// isStrictRFC3339 reports whether s matches strictRFC3339Pattern, hand-rolled to +// keep the conversion path off the regexp engine and its per-call cost. +func isStrictRFC3339(s string) bool { + // Shortest accepted form is "2006-01-02T15:04:05Z" (20 bytes): a 19-byte + // fixed-width date-time followed by at least a 'Z'/'z' zone. + if len(s) < 20 { + return false + } + // full-date "T" partial-time + if !isYear(s[0:4]) || !isChar(s[4], '-') || !isMonth(s[5:7]) || !isChar(s[7], '-') || !isDay(s[8:10]) || + !isChar(s[10], 't') || + !isHour(s[11:13]) || !isChar(s[13], ':') || !isMinute(s[14:16]) || !isChar(s[16], ':') || !isSecond(s[17:19]) { + return false + } + rest := s[19:] + // optional fractional seconds: "." 1*DIGIT + if rest[0] == '.' { + rest = rest[1:] + n := 0 + for n < len(rest) && isDigit(rest[n]) { + n++ + } + if n == 0 { + return false + } + rest = rest[n:] + } + // time-offset: "Z" or ("+" / "-") time-hour ":" time-minute + if len(rest) == 1 { + return isChar(rest[0], 'z') + } + if len(rest) == 6 && (rest[0] == '+' || rest[0] == '-') { + return isHour(rest[1:3]) && isChar(rest[3], ':') && isMinute(rest[4:6]) + } + return false +} + +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +// isChar reports whether got is want, case-insensitively; want must be lower case. +func isChar(got, want byte) bool { + g, w := rune(got), rune(want) + return g == w || unicode.ToLower(g) == w +} + +// inRange reports whether s is all decimal digits and its value lies in [lo, hi]. +func inRange(s string, lo, hi uint64) bool { + u, err := strconv.ParseUint(s, 10, 64) + return err == nil && u >= lo && u <= hi +} + +func isYear(s string) bool { return inRange(s, 0, 9999) } +func isMonth(s string) bool { return inRange(s, 1, 12) } +func isDay(s string) bool { return inRange(s, 1, 31) } +func isHour(s string) bool { return inRange(s, 0, 23) } +func isMinute(s string) bool { return inRange(s, 0, 59) } + +// isSecond permits 60 for a leap second. +func isSecond(s string) bool { return inRange(s, 0, 60) } + // Add implements traits.Adder.Add. func (t Timestamp) Add(other ref.Val) ref.Val { switch other.Type() { @@ -302,6 +377,9 @@ func timeZone(tz ref.Val, visitor timestampVisitor) timestampVisitor { if err != nil { return WrapErr(err) } + if min < 0 || min > 59 { + return WrapErr(fmt.Errorf("timezone offset minutes out of range [0, 59]: %s", val)) + } var offset int if string(val[0]) == "-" { offset = hr*60 - min diff --git a/vendor/github.com/google/cel-go/common/types/unknown.go b/vendor/github.com/google/cel-go/common/types/unknown.go index 9dd2b2579..f43aff18e 100644 --- a/vendor/github.com/google/cel-go/common/types/unknown.go +++ b/vendor/github.com/google/cel-go/common/types/unknown.go @@ -16,6 +16,7 @@ package types import ( "fmt" + "maps" "math" "reflect" "sort" @@ -181,6 +182,20 @@ func (u *Unknown) GetAttributeTrails(id int64) ([]*AttributeTrail, bool) { return trails, found } +// HasUnknownFunction returns whether any of the attribute trails contained within the unknown +// are unspecified. Unspecified attributes typically indicate an unresolved function call +// or operation, rather than a missing variable. +func (u *Unknown) HasUnknownFunction() bool { + for _, trails := range u.attributeTrails { + for _, t := range trails { + if t.variable == "" { + return true + } + } + } + return false +} + // Contains returns true if the input unknown is a subset of the current unknown. func (u *Unknown) Contains(other *Unknown) bool { for id, otherTrails := range other.attributeTrails { @@ -283,9 +298,7 @@ func MergeUnknowns(unk1, unk2 *Unknown) *Unknown { out := &Unknown{ attributeTrails: make(map[int64][]*AttributeTrail, len(unk1.attributeTrails)+len(unk2.attributeTrails)), } - for id, ats := range unk1.attributeTrails { - out.attributeTrails[id] = ats - } + maps.Copy(out.attributeTrails, unk1.attributeTrails) for id, ats := range unk2.attributeTrails { existing, found := out.attributeTrails[id] if !found { diff --git a/vendor/github.com/google/cel-go/interpreter/BUILD.bazel b/vendor/github.com/google/cel-go/interpreter/BUILD.bazel index 220e23d47..40ac2ba69 100644 --- a/vendor/github.com/google/cel-go/interpreter/BUILD.bazel +++ b/vendor/github.com/google/cel-go/interpreter/BUILD.bazel @@ -9,11 +9,13 @@ go_library( name = "go_default_library", srcs = [ "activation.go", + "async.go", "attribute_patterns.go", "attributes.go", "decorators.go", "dispatcher.go", "evalstate.go", + "frame.go", "interpretable.go", "interpreter.go", "optimizations.go", @@ -45,8 +47,10 @@ go_test( name = "go_default_test", srcs = [ "activation_test.go", + "async_test.go", "attribute_patterns_test.go", "attributes_test.go", + "frame_test.go", "interpreter_test.go", "prune_test.go", "runtimecost_test.go", @@ -63,6 +67,7 @@ go_test( "//common/operators:go_default_library", "//common/stdlib:go_default_library", "//common/types:go_default_library", + "//common/types/ref:go_default_library", "//parser:go_default_library", "//test:go_default_library", "//test/proto2pb:go_default_library", diff --git a/vendor/github.com/google/cel-go/interpreter/activation.go b/vendor/github.com/google/cel-go/interpreter/activation.go index dd40619ee..bc9296ed4 100644 --- a/vendor/github.com/google/cel-go/interpreter/activation.go +++ b/vendor/github.com/google/cel-go/interpreter/activation.go @@ -110,8 +110,9 @@ func (a *mapActivation) ResolveName(name string) (any, bool) { // hierarchicalActivation which implements Activation and contains a parent and // child activation. type hierarchicalActivation struct { - parent Activation - child Activation + parent Activation + child Activation + poolAllocated bool } // Parent implements the Activation interface method. @@ -127,10 +128,41 @@ func (a *hierarchicalActivation) ResolveName(name string) (any, bool) { return a.parent.ResolveName(name) } +// Unwrap returns the parent activation, stripping the local child scope. +// This allows global disambiguation to skip past locally introduced variables. +func (a *hierarchicalActivation) Unwrap() Activation { + return a.parent +} + +// IsLocalVariable reports whether the variable name is locally bound in the hierarchical activation. +func (a *hierarchicalActivation) IsLocalVariable(name string) bool { + if holder, ok := a.child.(localVariableHolder); ok { + if holder.IsLocalVariable(name) { + return true + } + } + if holder, ok := a.parent.(localVariableHolder); ok { + return holder.IsLocalVariable(name) + } + return false +} + +// AsPartialActivation checks the child first via direct type assertion (to +// avoid recursion through the folder → frame → hierarchicalActivation cycle), +// then walks the parent hierarchy via the free function. +func (a *hierarchicalActivation) AsPartialActivation() (PartialActivation, bool) { + if pv, ok := a.child.(partialActivationConverter); ok { + if p, ok := pv.AsPartialActivation(); ok { + return p, true + } + } + return AsPartialActivation(a.parent) +} + // NewHierarchicalActivation takes two activations and produces a new one which prioritizes // resolution in the child first and parent(s) second. func NewHierarchicalActivation(parent Activation, child Activation) Activation { - return &hierarchicalActivation{parent, child} + return &hierarchicalActivation{parent: parent, child: child, poolAllocated: false} } // NewPartialActivation returns an Activation which contains a list of AttributePattern values diff --git a/vendor/github.com/google/cel-go/interpreter/async.go b/vendor/github.com/google/cel-go/interpreter/async.go new file mode 100644 index 000000000..4e391196b --- /dev/null +++ b/vendor/github.com/google/cel-go/interpreter/async.go @@ -0,0 +1,530 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interpreter + +import ( + "context" + "encoding/binary" + "fmt" + "hash/fnv" + "math" + "sync" + "sync/atomic" + + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" +) + +// Async extension function support. +// +// CEL supports `types.Unknown` as a first-class value, and concurrent (async) function execution +// in CEL invokes a stub function which checks for the presence of an existing result which matches +// the function call and call arguments, or which records the 'unexecuted' function and call arguments +// for concurrent execution in a later phase if the result is `types.Unknown` and indicates the +// expression ids of the functions necessary to advance the execution. +// +// This call pattern is repeated iteratively until there are either no more functions to call or no +// progress is made toward resolving the unknowns. + +// AsyncObserver provides callbacks for monitoring the lifecycle of asynchronous function calls. +// +// Implementations must be safe for concurrent use: OnCallStarted is invoked from the evaluator +// goroutine when a call is launched, while OnCallFinished is invoked from the call's own goroutine +// when it completes. The two callbacks therefore run on different goroutines, and OnCallFinished +// callbacks for distinct calls may run concurrently with each other. +type AsyncObserver interface { + // OnCallStarted is called when an asynchronous function is first launched. + OnCallStarted(callID int64, function, overload string, args []ref.Val) + // OnCallFinished is called when an asynchronous function completes. + OnCallFinished(callID int64, function, overload string, res ref.Val) +} + +// AsyncCall describes a pending or completed asynchronous function call. +type AsyncCall interface { + // CallID returns the unique identifier for this async call invocation. + CallID() int64 + // Function returns the name of the function being called. + Function() string + // Overload returns the specific overload ID being invoked. + Overload() string +} + +// evalAsyncFunc is the planned Interpretable for an asynchronous function call. +type evalAsyncFunc struct { + id int64 + function string + overload string + args []InterpretableV2 + impl functions.AsyncOp +} + +// ID implements the Interpretable interface method. +func (fn *evalAsyncFunc) ID() int64 { + return fn.id +} + +// Function returns the name of the function being invoked. +func (fn *evalAsyncFunc) Function() string { + return fn.function +} + +// OverloadID returns the overload id of the function being invoked. +func (fn *evalAsyncFunc) OverloadID() string { + return fn.overload +} + +// Args returns the argument Interpretables for the function call. +func (fn *evalAsyncFunc) Args() []InterpretableV2 { + return fn.args +} + +// Eval implements the Interpretable interface method. +func (fn *evalAsyncFunc) Eval(vars Activation) ref.Val { + return fn.Exec(AsFrame(vars)) +} + +// Exec implements the InterpretableV2 interface method. +func (fn *evalAsyncFunc) Exec(frame *ExecutionFrame) ref.Val { + argVals := make([]ref.Val, len(fn.args)) + var unk *types.Unknown + for i, arg := range fn.args { + argVals[i] = arg.Exec(frame) + if types.IsError(argVals[i]) { + return argVals[i] + } + unk, _ = types.MaybeMergeUnknowns(argVals[i], unk) + } + if unk != nil { + return unk + } + result := frame.ComputeResult(fn.ID(), fn.Function(), fn.OverloadID(), fn.impl, argVals) + return types.LabelErrNode(fn.id, result) +} + +// asyncCallStateTracker manages async call states across re-evaluations of a single program. +type asyncCallStateTracker struct { + mu sync.RWMutex + // calls buckets call states by a composite hash of (node id, overload, string/int/double/uint/bool args). + // A single AST node id may host many concurrently-live calls when it is evaluated inside a + // comprehension (once per element with different arguments), so each bucket may hold more + // than one state. The exact match within a bucket is resolved via asyncCallState.matches, + // which applies CEL's full equality semantics to the arguments. + calls map[uint64][]*asyncCallState + callsByID map[int64]*asyncCallState + nextCallID atomic.Int64 +} + +func newAsyncCallStateTracker() *asyncCallStateTracker { + return &asyncCallStateTracker{ + calls: make(map[uint64][]*asyncCallState), + callsByID: make(map[int64]*asyncCallState), + } +} + +var ( + hashZeroMarker = []byte{0} + hashStringMarker = []byte{'s'} + hashBoolTrueMarker = []byte{'b', 1} + hashBoolFalseMarker = []byte{'b', 0} + hashNumberMarker = []byte{'n'} + hashDefaultMarker = []byte{'x'} +) + +// hashCall computes the composite bucket key for an async call. +// +// Only string, int, double, uint, and bool argument values contribute to the hash. More complex types +// rely on a richer notion of equivalence (e.g. unordered maps, proto equality, custom types) +// that a byte-level hash cannot capture safely, so they are intentionally excluded from the key +// and are instead disambiguated within the bucket by asyncCallState.matches. +func hashCall(id int64, overload string, args []ref.Val) uint64 { + h := fnv.New64a() + var idBuf [8]byte + binary.LittleEndian.PutUint64(idBuf[:], uint64(id)) + h.Write(idBuf[:]) + h.Write([]byte(overload)) + h.Write(hashZeroMarker) + for _, arg := range args { + switch v := arg.(type) { + case types.String: + h.Write(hashStringMarker) + h.Write([]byte(string(v))) + case types.Bool: + if bool(v) { + h.Write(hashBoolTrueMarker) + } else { + h.Write(hashBoolFalseMarker) + } + case types.Int: + h.Write(hashNumberMarker) + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], math.Float64bits(float64(v))) + h.Write(buf[:]) + case types.Uint: + h.Write(hashNumberMarker) + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], math.Float64bits(float64(v))) + h.Write(buf[:]) + case types.Double: + h.Write(hashNumberMarker) + if math.IsNaN(float64(v)) { + h.Write([]byte("NaN")) + h.Write(hashZeroMarker) + continue + } + // Normalize -0.0 to 0.0. Go will treat -0.0 as 0.0 at compile time, + // but the function math.Copysign(0.0, -1.0) can be used to test the -0.0 case. + if v == types.Double(0.0) && math.Signbit(float64(v)) { + v = types.Double(0.0) + } + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], math.Float64bits(float64(v))) + h.Write(buf[:]) + default: + // Value intentionally omitted; bucket membership falls back to matches. + h.Write(hashDefaultMarker) + } + // Separator to avoid cross-argument collisions, e.g. ("a", "bc") vs ("ab", "c"). + h.Write(hashZeroMarker) + } + return h.Sum64() +} + +// findInBucket returns the call state in the bucket matching the same node id and call identity, +// or nil if no match is present. +func findInBucket(bucket []*asyncCallState, id int64, function, overload string, args []ref.Val) *asyncCallState { + for _, acs := range bucket { + if acs.matches(id, function, overload, args) { + return acs + } + } + return nil +} + +// getOrCreate returns the existing call state for the (node id, args) tuple, or registers and +// returns a new one. A newly registered call is assigned a unique callID and counted as pending. +func (t *asyncCallStateTracker) getOrCreate(id int64, function, overload string, argVals []ref.Val, impl functions.AsyncOp, gate *asyncGate) *asyncCallState { + key := hashCall(id, overload, argVals) + + t.mu.RLock() + acs := findInBucket(t.calls[key], id, function, overload, argVals) + t.mu.RUnlock() + if acs != nil { + return acs + } + + t.mu.Lock() + defer t.mu.Unlock() + // Check again in case it was created while waiting for the lock. + if acs := findInBucket(t.calls[key], id, function, overload, argVals); acs != nil { + return acs + } + + // Assign a new unique call ID for this async call. + acs = newAsyncCallState(id, function, overload, argVals, impl) + callID := t.nextCallID.Add(1) + acs.callID = callID + acs.gate = gate + t.calls[key] = append(t.calls[key], acs) + t.callsByID[callID] = acs + return acs +} + +func (t *asyncCallStateTracker) getByID(callID int64) *asyncCallState { + t.mu.RLock() + defer t.mu.RUnlock() + return t.callsByID[callID] +} + +func newAsyncCallState(id int64, function, overload string, argVals []ref.Val, impl functions.AsyncOp) *asyncCallState { + return &asyncCallState{ + id: id, + function: function, + overload: overload, + argVals: argVals, + impl: impl, + } +} + +// asyncCallState tracks the result of a single async function call across multiple re-evaluations. +type asyncCallState struct { + id int64 // AST expression node ID where the call is defined. + callID int64 // Unique incremental tracking ID assigned to this call. + function string + overload string + argVals []ref.Val + impl functions.AsyncOp + + mu sync.RWMutex + started bool + result ref.Val + + gate *asyncGate +} + +// CallID returns the unique identifier for this async call invocation. +func (acs *asyncCallState) CallID() int64 { + return acs.callID +} + +// Function returns the name of the function being called. +func (acs *asyncCallState) Function() string { + return acs.function +} + +// Overload returns the specific overload ID being invoked. +func (acs *asyncCallState) Overload() string { + return acs.overload +} + +// ResultOrUnknown returns the cached result if the call has completed, an Unknown +// with the call ID if pending, or nil if the call has not been started. +func (acs *asyncCallState) ResultOrUnknown() ref.Val { + if acs == nil { + return nil + } + acs.mu.RLock() + defer acs.mu.RUnlock() + if acs.result == nil && acs.started { + return types.NewUnknown(acs.callID, nil) + } + return acs.result +} + +// SetResult sets the completed result for an asynchronous function call. +func (acs *asyncCallState) SetResult(res ref.Val) { + if acs == nil { + return + } + acs.mu.Lock() + defer acs.mu.Unlock() + acs.result = res +} + +// launch returns a call's cached result, or starts the call (subject to the launch limiter) and +// returns an Unknown referencing its callID while the result is pending. +// +// Admission control: when a concurrency semaphore is configured, a launch slot is reserved with a +// non-blocking send. If no slot is free the call is left unstarted and an Unknown is returned; the +// call is retried on a later re-evaluation pass once an in-flight call completes and frees a slot. +// The reservation is non-blocking on purpose — the evaluator runs on a single goroutine, and +// blocking it here while completing calls block on an undrained completion channel would deadlock. +// The slot is held by the launched goroutine and released when it exits, so the number of live +// async goroutines is bounded by the semaphore capacity. +func (t *asyncCallStateTracker) launch(ctx context.Context, acs *asyncCallState, observer AsyncObserver) ref.Val { + if res := acs.ResultOrUnknown(); res != nil { + return res + } + gate := acs.gate + if !gate.TryAcquire() { + return types.NewUnknown(acs.callID, nil) + } + acs.mu.Lock() + if acs.started || acs.result != nil { + // Defensive: the evaluator is single-threaded so this should not happen, but if it does, + // return the reserved slot rather than leak it. + acs.mu.Unlock() + gate.Release() + return types.NewUnknown(acs.callID, nil) + } + acs.started = true + acs.mu.Unlock() + + if observer != nil { + observer.OnCallStarted(acs.callID, acs.function, acs.overload, acs.argVals) + } + go func() { + defer func() { + if observer != nil { + observer.OnCallFinished(acs.callID, acs.function, acs.overload, acs.ResultOrUnknown()) + } + gate.Complete(ctx, acs.callID) + }() + + ch := acs.impl(ctx, acs.argVals...) + // Early terminate with a CEL error when an implementation returns an empty channel. + if ch == nil { + acs.SetResult(types.NewErrFromString( + fmt.Sprintf("function %s returned an empty channel", acs.function))) + return + } + // Wait for the async computation to finish or for the context to be cancelled. + select { + case r, ok := <-ch: + if !ok { + acs.SetResult(types.NewErrFromString( + fmt.Sprintf("function %s returned an empty channel", acs.function))) + return + } + acs.SetResult(r) + case <-ctx.Done(): + // Evaluation context cancelled before the async operation completed. + acs.SetResult(types.WrapErr(context.Cause(ctx))) + } + }() + return types.NewUnknown(acs.callID, nil) +} + +// matches reports whether two call states refer to the same function, overload, and arguments. +func (acs *asyncCallState) matches(id int64, function, overload string, args []ref.Val) bool { + if acs == nil { + return false + } + if acs.id != id || acs.function != function || acs.overload != overload { + return false + } + if len(acs.argVals) != len(args) { + return false + } + for i, v := range acs.argVals { + otherV := args[i] + if types.Equal(v, otherV) == types.True { + continue + } + if n, ok := v.(types.Double); ok { + // Treat NaN as equivalent for the sake of function dispatch equality. + if otherN, ok := otherV.(types.Double); ok && math.IsNaN(float64(n)) && math.IsNaN(float64(otherN)) { + continue + } + } + return false + } + return true +} + +// trackerShrinkThreshold is the entry count above which a released tracker's maps are reallocated +// rather than cleared in place, so the pool does not retain a large backing array indefinitely. +const trackerShrinkThreshold = 1024 + +// asyncCallStateTrackerPool provides a synchronized pool of asyncCallStateTrackers. +type asyncCallTrackerPool struct { + sync.Pool +} + +func (pool *asyncCallTrackerPool) create() *asyncCallStateTracker { + return pool.Get().(*asyncCallStateTracker) +} + +func (pool *asyncCallTrackerPool) release(tracker *asyncCallStateTracker) { + if tracker == nil { + return + } + tracker.mu.Lock() + // Clearing with delete reuses the backing arrays, which is ideal for the common case but pins + // a large allocation in the pool after a wide fan-out (e.g. an async call over a big list). + // Past a threshold, reallocate so the high-water-mark memory is released to the GC instead of + // being retained by the pooled tracker. + if len(tracker.calls) > trackerShrinkThreshold || len(tracker.callsByID) > trackerShrinkThreshold { + tracker.calls = make(map[uint64][]*asyncCallState) + tracker.callsByID = make(map[int64]*asyncCallState) + } else { + for k := range tracker.calls { + delete(tracker.calls, k) + } + for k := range tracker.callsByID { + delete(tracker.callsByID, k) + } + } + tracker.nextCallID.Store(0) + tracker.mu.Unlock() + pool.Pool.Put(tracker) +} + +func newAsyncCallTrackerPool() *asyncCallTrackerPool { + return &asyncCallTrackerPool{ + Pool: sync.Pool{ + New: func() any { + return newAsyncCallStateTracker() + }, + }, + } +} + +var asyncCallStateTrackerPool = newAsyncCallTrackerPool() + +// asyncGate coordinates async call admission control and completion signaling. +type asyncGate struct { + semaphore chan struct{} + completions chan<- int64 + activeCalls atomic.Int32 +} + +func newAsyncGate(maxConcurrency int, completions chan<- int64) *asyncGate { + var sem chan struct{} + if maxConcurrency > 0 { + sem = make(chan struct{}, maxConcurrency) + } + return &asyncGate{ + semaphore: sem, + completions: completions, + } +} + +// TryAcquire attempts to acquire a concurrency slot and increments the active calls count. +func (g *asyncGate) TryAcquire() bool { + if g == nil { + return true + } + if g.semaphore != nil { + select { + case g.semaphore <- struct{}{}: + default: + return false + } + } + g.activeCalls.Add(1) + return true +} + +// Release releases a concurrency slot and decrements the active calls count (used for defensive recovery). +func (g *asyncGate) Release() { + if g == nil { + return + } + if g.semaphore != nil { + select { + case <-g.semaphore: + default: + } + } + g.activeCalls.Add(-1) +} + +// Complete releases a concurrency slot and notifies completions. +func (g *asyncGate) Complete(ctx context.Context, callID int64) { + if g == nil { + return + } + g.Release() + + if g.completions != nil { + // Prioritize context cancellation to prevent racy completion signals. + if ctx.Err() != nil { + return + } + select { + case g.completions <- callID: + case <-ctx.Done(): + } + } +} + +// ActiveCalls returns the number of active asynchronous calls. +func (g *asyncGate) ActiveCalls() int { + if g == nil { + return 0 + } + return int(g.activeCalls.Load()) +} diff --git a/vendor/github.com/google/cel-go/interpreter/attribute_patterns.go b/vendor/github.com/google/cel-go/interpreter/attribute_patterns.go index 41ca5cd21..bbaca5226 100644 --- a/vendor/github.com/google/cel-go/interpreter/attribute_patterns.go +++ b/vendor/github.com/google/cel-go/interpreter/attribute_patterns.go @@ -255,6 +255,9 @@ func (fac *partialAttributeFactory) matchesUnknownPatterns( patterns := vars.UnknownAttributePatterns() candidateIndices := map[int]struct{}{} for _, variable := range variableNames { + if holder, ok := vars.(localVariableHolder); ok && holder.IsLocalVariable(variable) { + continue + } for i, pat := range patterns { if pat.VariableMatches(variable) { if len(qualifiers) == 0 { diff --git a/vendor/github.com/google/cel-go/interpreter/attributes.go b/vendor/github.com/google/cel-go/interpreter/attributes.go index 6b8b5c1b6..26d8eb0f3 100644 --- a/vendor/github.com/google/cel-go/interpreter/attributes.go +++ b/vendor/github.com/google/cel-go/interpreter/attributes.go @@ -190,7 +190,7 @@ func (r *attrFactory) AbsoluteAttribute(id int64, names ...string) NamespacedAtt func (r *attrFactory) ConditionalAttribute(id int64, expr Interpretable, t, f Attribute) Attribute { return &conditionalAttribute{ id: id, - expr: expr, + expr: adaptToV2(expr), truthy: t, falsy: f, adapter: r.adapter, @@ -225,7 +225,7 @@ func (r *attrFactory) MaybeAttribute(id int64, name string) Attribute { func (r *attrFactory) RelativeAttribute(id int64, operand Interpretable) Attribute { return &relativeAttribute{ id: id, - operand: operand, + operand: adaptToV2(operand), qualifiers: []Qualifier{}, adapter: r.adapter, fac: r, @@ -384,7 +384,7 @@ func (a *absoluteAttribute) Resolve(vars Activation) (any, error) { type conditionalAttribute struct { id int64 - expr Interpretable + expr InterpretableV2 truthy Attribute falsy Attribute adapter types.Adapter @@ -571,7 +571,7 @@ func (a *maybeAttribute) String() string { type relativeAttribute struct { id int64 - operand Interpretable + operand InterpretableV2 qualifiers []Qualifier adapter types.Adapter fac AttributeFactory @@ -964,9 +964,11 @@ func (q *intQualifier) qualifyInternal(vars Activation, obj any, presenceTest, p } case map[int32]any: isMap = true - obj, isKey := o[int32(i)] - if isKey { - return obj, true, nil + if i32 := int32(i); int64(i32) == i { + obj, isKey := o[i32] + if isKey { + return obj, true, nil + } } case map[int64]any: isMap = true @@ -1089,9 +1091,11 @@ func (q *uintQualifier) qualifyInternal(vars Activation, obj any, presenceTest, return obj, true, nil } case map[uint32]any: - obj, isKey := o[uint32(u)] - if isKey { - return obj, true, nil + if u32 := uint32(u); uint64(u32) == u { + obj, isKey := o[u32] + if isKey { + return obj, true, nil + } } case map[uint64]any: obj, isKey := o[u] @@ -1301,7 +1305,7 @@ func applyQualifiers(vars Activation, obj any, qualifiers []Qualifier) (any, boo if !optObj.HasValue() { return optObj, false, nil } - obj = optObj.GetValue().Value() + obj = optObj.GetValue() } var err error diff --git a/vendor/github.com/google/cel-go/interpreter/decorators.go b/vendor/github.com/google/cel-go/interpreter/decorators.go index 502db35fc..9c973664a 100644 --- a/vendor/github.com/google/cel-go/interpreter/decorators.go +++ b/vendor/github.com/google/cel-go/interpreter/decorators.go @@ -25,9 +25,13 @@ import ( // Interpretable expression nodes at construction time. type InterpretableDecorator func(Interpretable) (Interpretable, error) +// InterpretableDecoratorV2 is a functional interface for decorating or replacing +// InterpretableV2 expression nodes at construction time. +type InterpretableDecoratorV2 func(InterpretableV2) (InterpretableV2, error) + // decObserveEval records evaluation state into an EvalState object. -func decObserveEval(observer EvalObserver) InterpretableDecorator { - return func(i Interpretable) (Interpretable, error) { +func decObserveEval(observer EvalObserver) InterpretableDecoratorV2 { + return func(i InterpretableV2) (InterpretableV2, error) { switch inst := i.(type) { case *evalWatch, *evalWatchAttr, *evalWatchConst, *evalWatchConstructor: // these instruction are already watching, return straight-away. @@ -49,8 +53,8 @@ func decObserveEval(observer EvalObserver) InterpretableDecorator { }, nil default: return &evalWatch{ - Interpretable: i, - observer: observer, + InterpretableV2: i, + observer: observer, }, nil } } @@ -58,8 +62,8 @@ func decObserveEval(observer EvalObserver) InterpretableDecorator { // decInterruptFolds creates an intepretable decorator which marks comprehensions as interruptable // where the interrupt state is communicated via a hidden variable on the Activation. -func decInterruptFolds() InterpretableDecorator { - return func(i Interpretable) (Interpretable, error) { +func decInterruptFolds() InterpretableDecoratorV2 { + return func(i InterpretableV2) (InterpretableV2, error) { fold, ok := i.(*evalFold) if !ok { return i, nil @@ -70,8 +74,8 @@ func decInterruptFolds() InterpretableDecorator { } // decDisableShortcircuits ensures that all branches of an expression will be evaluated, no short-circuiting. -func decDisableShortcircuits() InterpretableDecorator { - return func(i Interpretable) (Interpretable, error) { +func decDisableShortcircuits() InterpretableDecoratorV2 { + return func(i InterpretableV2) (InterpretableV2, error) { switch expr := i.(type) { case *evalOr: return &evalExhaustiveOr{ @@ -104,8 +108,8 @@ func decDisableShortcircuits() InterpretableDecorator { // conditionally precomputing the result. // - build list and map values with constant elements. // - convert 'in' operations to set membership tests if possible. -func decOptimize() InterpretableDecorator { - return func(i Interpretable) (Interpretable, error) { +func decOptimize() InterpretableDecoratorV2 { + return func(i InterpretableV2) (InterpretableV2, error) { switch inst := i.(type) { case *evalList: return maybeBuildListLiteral(i, inst) @@ -124,7 +128,7 @@ func decOptimize() InterpretableDecorator { } // decRegexOptimizer compiles regex pattern string constants. -func decRegexOptimizer(regexOptimizations ...*RegexOptimization) InterpretableDecorator { +func decRegexOptimizer(regexOptimizations ...*RegexOptimization) InterpretableDecoratorV2 { functionMatchMap := make(map[string]*RegexOptimization) overloadMatchMap := make(map[string]*RegexOptimization) for _, m := range regexOptimizations { @@ -134,7 +138,7 @@ func decRegexOptimizer(regexOptimizations ...*RegexOptimization) InterpretableDe } } - return func(i Interpretable) (Interpretable, error) { + return func(i InterpretableV2) (InterpretableV2, error) { call, ok := i.(InterpretableCall) if !ok { return i, nil @@ -165,7 +169,7 @@ func decRegexOptimizer(regexOptimizations ...*RegexOptimization) InterpretableDe } } -func maybeOptimizeConstUnary(i Interpretable, call InterpretableCall) (Interpretable, error) { +func maybeOptimizeConstUnary(i InterpretableV2, call InterpretableCall) (InterpretableV2, error) { args := call.Args() if len(args) != 1 { return i, nil @@ -181,7 +185,7 @@ func maybeOptimizeConstUnary(i Interpretable, call InterpretableCall) (Interpret return NewConstValue(call.ID(), val), nil } -func maybeBuildListLiteral(i Interpretable, l *evalList) (Interpretable, error) { +func maybeBuildListLiteral(i InterpretableV2, l *evalList) (InterpretableV2, error) { for _, elem := range l.elems { _, isConst := elem.(InterpretableConst) if !isConst { @@ -191,7 +195,7 @@ func maybeBuildListLiteral(i Interpretable, l *evalList) (Interpretable, error) return NewConstValue(l.ID(), l.Eval(EmptyActivation())), nil } -func maybeBuildMapLiteral(i Interpretable, mp *evalMap) (Interpretable, error) { +func maybeBuildMapLiteral(i InterpretableV2, mp *evalMap) (InterpretableV2, error) { for idx, key := range mp.keys { _, isConst := key.(InterpretableConst) if !isConst { @@ -209,7 +213,7 @@ func maybeBuildMapLiteral(i Interpretable, mp *evalMap) (Interpretable, error) { // test if the following conditions are true: // - the list is a constant with homogeneous element types. // - the elements are all of primitive type. -func maybeOptimizeSetMembership(i Interpretable, inlist InterpretableCall) (Interpretable, error) { +func maybeOptimizeSetMembership(i InterpretableV2, inlist InterpretableCall) (InterpretableV2, error) { args := inlist.Args() lhs := args[0] rhs := args[1] diff --git a/vendor/github.com/google/cel-go/interpreter/frame.go b/vendor/github.com/google/cel-go/interpreter/frame.go new file mode 100644 index 000000000..20ab313c8 --- /dev/null +++ b/vendor/github.com/google/cel-go/interpreter/frame.go @@ -0,0 +1,445 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interpreter + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" +) + +// evalContext contains the stateful information needed for a single evaluation. +// +// This state is shared across all frames within a single evaluation, including +// child frames created for comprehension blocks. +type evalContext struct { + // interrupt exposes a callback channel for cancellation. + interrupt <-chan struct{} + + // interruptCheckCount is the number of times the interrupt channel has been checked. + interruptCheckCount atomic.Uint64 + + // interruptCheckFrequency is the frequency at which the interrupt channel is checked. + interruptCheckFrequency uint + + // interrupted indicates whether the evaluation has been interrupted. + interrupted atomic.Bool + + // state provides the context for tracking the evaluation state. + state EvalState + + // costs provides the context for tracking the evaluation costs. + costs *CostTracker + + // ctx is the context for async call implementations to use. + ctx context.Context + + // cancel cancels the context when the evaluation is finished. + cancel context.CancelFunc + + // asyncCalls tracks the state of async call invocations across re-evaluations. + asyncCalls *asyncCallStateTracker + + // gate coordinates async call admission control and completion signaling. + gate *asyncGate + + // observer for monitoring async calls. + observer AsyncObserver +} + +// ExecutionFrame provides the context for a single evaluation of an expression. +// +// The execution frame must not be stored in any fashion as its lifecycle is completely +// controlled by the CEL evaluation process. +type ExecutionFrame struct { + // Activation provides the context for resolving variables by name. + Activation + + // parent provides the context for parent scopes (used for comprehension iterators). + parent *ExecutionFrame + + // ctx provides the shared evaluation state across frames. + ctx *evalContext +} + +// NewExecutionFrame creates a new execution frame from the pool. +func NewExecutionFrame(input any) (*ExecutionFrame, error) { + f := frameStack.Get().(*ExecutionFrame) + switch v := input.(type) { + case Activation: + f.Activation = v + case map[string]any: + f.Activation = activationInput.create(v) + default: + return nil, fmt.Errorf("invalid input, wanted Activation or map[string]any, got: (%T)%v", input, input) + } + return f, nil +} + +// SetContext sets the context for the execution frame. +func (f *ExecutionFrame) SetContext(ctx context.Context, interruptCheckFrequency uint) error { + if f.parent != nil { + return errors.New("SetContext() called on child frame") + } + if f.ctx != nil { + return errors.New("SetContext() called more than once") + } + f.ctx = evalContextPool.Get().(*evalContext) + f.ctx.ctx, f.ctx.cancel = context.WithCancel(ctx) + f.ctx.asyncCalls = asyncCallStateTrackerPool.create() + f.ctx.gate = &asyncGate{} + f.ctx.interrupt = ctx.Done() + f.ctx.interruptCheckFrequency = interruptCheckFrequency + f.ctx.interruptCheckCount.Store(0) + f.ctx.interrupted.Store(false) + return nil +} + +// Close releases the resources held by the execution frame and returns it to the pool. +func (f *ExecutionFrame) Close() { + if f.parent == nil && f.ctx != nil { + if f.ctx.cancel != nil { + f.ctx.cancel() + f.ctx.cancel = nil + } + f.ctx.ctx = nil + f.ctx.gate = nil + asyncCallStateTrackerPool.release(f.ctx.asyncCalls) + f.ctx.asyncCalls = nil + f.ctx.observer = nil + f.ctx.interrupt = nil + f.ctx.state = nil + f.ctx.costs = nil + f.ctx.interrupted.Store(false) + f.ctx.interruptCheckCount.Store(0) + f.ctx.interruptCheckFrequency = 0 + evalContextPool.Put(f.ctx) + } + f.ctx = nil + f.parent = nil + if f.Activation != nil { + switch a := f.Activation.(type) { + case *hierarchicalActivation: + if child, ok := a.child.(*inputActivation); ok { + activationInput.release(child) + } + activationStack.release(a) + case *inputActivation: + activationInput.release(a) + } + f.Activation = nil + frameStack.Put(f) + } +} + +// Push pushes the given activation onto the activation stack and returns the new frame. +// +// This operation is internal to the interpreter and is used to handle comprehension +// scoping. The child frame inherits the shared evalContext from the parent. +func (f *ExecutionFrame) Push(activation Activation) *ExecutionFrame { + child := frameStack.Get().(*ExecutionFrame) + child.parent = f + child.ctx = f.ctx + child.Activation = activationStack.create(f.Activation, activation) + return child +} + +// Pop returns the parent frame, releasing the current frame back to the pool. +func (f *ExecutionFrame) Pop() *ExecutionFrame { + if f.parent == nil { + return f + } + parent := f.parent + activationStack.release(f.Activation) + f.Activation = nil + f.parent = nil + f.ctx = nil + frameStack.Put(f) + return parent +} + +// ResolveName implements the Activation interface by proxying to the internal activation. +func (f *ExecutionFrame) ResolveName(name string) (any, bool) { + return f.Activation.ResolveName(name) +} + +// Parent implements the Activation interface by proxying to the internal activation. +func (f *ExecutionFrame) Parent() Activation { + return f.Activation.Parent() +} + +// AsPartialActivation implements the PartialActivation interface by proxying to the internal activation. +func (f *ExecutionFrame) AsPartialActivation() (PartialActivation, bool) { + return AsPartialActivation(f.Activation) +} + +// Unwrap returns the internal activation. +func (f *ExecutionFrame) Unwrap() Activation { + return f.Activation +} + +// IsLocalVariable reports whether the variable name is locally bound in the frame. +func (f *ExecutionFrame) IsLocalVariable(name string) bool { + if holder, ok := f.Activation.(localVariableHolder); ok { + if holder.IsLocalVariable(name) { + return true + } + } + // Search parent scopes + if f.parent != nil { + return f.parent.IsLocalVariable(name) + } + return false +} + +// CheckInterrupt returns whether the evaluation has been interrupted. +func (f *ExecutionFrame) CheckInterrupt() bool { + if f.ctx == nil { + return false + } + if f.ctx.interrupted.Load() { + return true + } + count := f.ctx.interruptCheckCount.Add(1) + if f.ctx.interruptCheckFrequency > 0 && count%uint64(f.ctx.interruptCheckFrequency) == 0 { + select { + case <-f.ctx.interrupt: + f.ctx.interrupted.Store(true) + return true + default: + return false + } + } + return false +} + +// ComputeResult tracks and computes the result of the given asynchronous function. +// +// The first invocation for a given (node id, args) tuple registers the call state and returns an +// Unknown which references the call's unique callID. Subsequent invocations return the cached +// result once the call has completed. Launching background execution is deferred to post-execution +// dispatch via DispatchPendingAsyncCalls. +func (f *ExecutionFrame) ComputeResult(id int64, function, overload string, impl functions.AsyncOp, argVals []ref.Val) ref.Val { + if f.ctx == nil || f.ctx.asyncCalls == nil { + return types.NewErrWithNodeID(id, "asynchronous function calls require concurrent evaluation and cannot be resolved by a synchronous Eval") + } + t := f.ctx.asyncCalls + acs := t.getOrCreate(id, function, overload, argVals, impl, f.ctx.gate) + if res := acs.ResultOrUnknown(); res != nil { + return res + } + return types.NewUnknown(acs.callID, nil) +} + +// DispatchPendingAsyncCalls launches pending asynchronous calls for the specified required call IDs. +func (f *ExecutionFrame) DispatchPendingAsyncCalls(callIDs []int64) { + if f.ctx == nil || f.ctx.asyncCalls == nil { + return + } + t := f.ctx.asyncCalls + for _, callID := range callIDs { + if acs := t.getByID(callID); acs != nil { + t.launch(f.ctx.ctx, acs, f.ctx.observer) + } + } +} + +// ActiveAsyncCalls returns the number of async function calls that have been launched +// but whose completions have not yet been drained. +func (f *ExecutionFrame) ActiveAsyncCalls() int { + if f.ctx == nil || f.ctx.gate == nil { + return 0 + } + return f.ctx.gate.ActiveCalls() +} + +// AsyncCall returns the state of an async call by its callID, or nil if not found. +func (f *ExecutionFrame) AsyncCall(callID int64) AsyncCall { + if f.ctx == nil || f.ctx.asyncCalls == nil { + return nil + } + acs := f.ctx.asyncCalls.getByID(callID) + if acs == nil { + return nil + } + return acs +} + +// SetCompletions configures a channel to receive callIDs when asynchronous evaluations finish. +func (f *ExecutionFrame) SetCompletions(ch chan<- int64) error { + if f.ctx == nil { + return errors.New("asynchronous evaluation options require the execution frame to have a context configured") + } + f.ctx.gate.completions = ch + return nil +} + +// SetAsyncObserver sets the observer for monitoring asynchronous function calls. +func (f *ExecutionFrame) SetAsyncObserver(observer AsyncObserver) error { + if f.ctx == nil { + return errors.New("asynchronous evaluation options require the execution frame to have a context configured") + } + f.ctx.observer = observer + return nil +} + +// SetAsyncMaxConcurrency sets the maximum concurrency for asynchronous function calls. +// +// A non-positive value indicates that concurrency is unbounded. +func (f *ExecutionFrame) SetAsyncMaxConcurrency(n int) error { + if f.ctx == nil { + return errors.New("asynchronous evaluation options require the execution frame to have a context configured") + } + if n > 0 { + f.ctx.gate.semaphore = make(chan struct{}, n) + } else { + f.ctx.gate.semaphore = nil + } + return nil +} + +// frameStack provides a synchronized pool of ExecutionFrames. +var frameStack = &sync.Pool{ + New: func() any { + return &ExecutionFrame{} + }, +} + +// evalContextPool provides a synchronized pool of evalContexts. +var evalContextPool = &sync.Pool{ + New: func() any { + return &evalContext{} + }, +} + +type activationStackPool struct { + sync.Pool +} + +func (pool *activationStackPool) create(parent, child Activation) Activation { + h := pool.Get().(*hierarchicalActivation) + h.child = child + h.parent = parent + h.poolAllocated = true + return h +} + +func (pool *activationStackPool) release(activation Activation) { + h, ok := activation.(*hierarchicalActivation) + if !ok || !h.poolAllocated { + return + } + h.parent = nil + h.child = nil + pool.Pool.Put(h) +} + +func newActivationStackPool() *activationStackPool { + return &activationStackPool{ + Pool: sync.Pool{ + New: func() any { + return &hierarchicalActivation{} + }, + }, + } +} + +type inputActivation struct { + vars map[string]any + lazyVars map[string]any +} + +// ResolveName looks up the value of the input variable name, if found. +// +// Lazy bindings may be supplied within the map-based input in either of the following forms: +// - func() any +// - func() ref.Val +// +// The lazy binding will only be invoked once per evaluation. +// +// Values which are not represented as ref.Val types on input may be adapted to a ref.Val using +// the types.Adapter configured in the environment. +func (a *inputActivation) ResolveName(name string) (any, bool) { + v, found := a.vars[name] + if !found { + return nil, false + } + switch obj := v.(type) { + case func() ref.Val: + if resolved, found := a.lazyVars[name]; found { + return resolved, true + } + lazy := obj() + a.lazyVars[name] = lazy + return lazy, true + case func() any: + if resolved, found := a.lazyVars[name]; found { + return resolved, true + } + lazy := obj() + a.lazyVars[name] = lazy + return lazy, true + default: + return obj, true + } +} + +// Parent implements the Activation interface +func (a *inputActivation) Parent() Activation { + return nil +} + +func newActivationInputPool() *activationInputPool { + return &activationInputPool{ + Pool: sync.Pool{ + New: func() any { + return &inputActivation{ + lazyVars: make(map[string]any), + } + }, + }, + } +} + +type activationInputPool struct { + sync.Pool +} + +// create initializes a pooled Activation object with the map input. +func (p *activationInputPool) create(vars map[string]any) *inputActivation { + a := p.Pool.Get().(*inputActivation) + a.vars = vars + return a +} + +func (p *activationInputPool) release(value any) { + a := value.(*inputActivation) + for k := range a.lazyVars { + delete(a.lazyVars, k) + } + a.vars = nil + p.Pool.Put(a) +} + +var ( + activationStack = newActivationStackPool() + activationInput = newActivationInputPool() +) diff --git a/vendor/github.com/google/cel-go/interpreter/interpretable.go b/vendor/github.com/google/cel-go/interpreter/interpretable.go index 50e66d637..906c4f805 100644 --- a/vendor/github.com/google/cel-go/interpreter/interpretable.go +++ b/vendor/github.com/google/cel-go/interpreter/interpretable.go @@ -26,20 +26,53 @@ import ( "github.com/google/cel-go/common/types/traits" ) -// Interpretable can accept a given Activation and produce a value along with -// an accompanying EvalState which can be used to inspect whether additional -// data might be necessary to complete the evaluation. +// Interpretable evaluates an Activation and produces a value. type Interpretable interface { // ID value corresponding to the expression node. ID() int64 - // Eval an Activation to produce an output. + // Eval evaluates an Activation and produces an output. Eval(activation Activation) ref.Val } +// InterpretableV2 evaluates an ExecutionFrame and produces a value. +// +// The ExecutionFrame should not be stored and should always be passed as the first +// argument to any function as it behaves like Golang's context.Context. +type InterpretableV2 interface { + Interpretable + + // Exec evaluates the expression within the given ExecutionFrame. + Exec(frame *ExecutionFrame) ref.Val +} + +// adaptToV2 adapts a V1 Interpretable implementation to the V2 interface. +// +// This adapter is used to bridge the legacy Interpretable interface to the +// modern InterpretableV2 interface, providing a shim that allows the use of +// both interfaces in the same system. +func adaptToV2(i Interpretable) InterpretableV2 { + switch v := i.(type) { + case InterpretableV2: + return v + default: + return &v1Adapter{Interpretable: v} + } +} + +// v1Adapter handles bridging a V1 Interpretable implementation to the V2 interface. +type v1Adapter struct { + Interpretable +} + +// Exec implements the InterpretableV2 interface method. +func (a *v1Adapter) Exec(f *ExecutionFrame) ref.Val { + return a.Eval(f) +} + // InterpretableConst interface for tracking whether the Interpretable is a constant value. type InterpretableConst interface { - Interpretable + InterpretableV2 // Value returns the constant value of the instruction. Value() ref.Val @@ -47,7 +80,7 @@ type InterpretableConst interface { // InterpretableAttribute interface for tracking whether the Interpretable is an attribute. type InterpretableAttribute interface { - Interpretable + InterpretableV2 // Attr returns the Attribute value. Attr() Attribute @@ -81,7 +114,7 @@ type InterpretableAttribute interface { // InterpretableCall interface for inspecting Interpretable instructions related to function calls. type InterpretableCall interface { - Interpretable + InterpretableV2 // Function returns the function name as it appears in text or mangled operator name as it // appears in the operators.go file. @@ -94,16 +127,16 @@ type InterpretableCall interface { // Args returns the normalized arguments to the function overload. // For receiver-style functions, the receiver target is arg 0. - Args() []Interpretable + Args() []InterpretableV2 } // InterpretableConstructor interface for inspecting Interpretable instructions that initialize a list, map // or struct. type InterpretableConstructor interface { - Interpretable + InterpretableV2 // InitVals returns all the list elements, map key and values or struct field values. - InitVals() []Interpretable + InitVals() []InterpretableV2 // Type returns the type constructed. Type() ref.Type @@ -112,18 +145,23 @@ type InterpretableConstructor interface { // ObservableInterpretable is an Interpretable which supports stateful observation, such as tracing // or cost-tracking. type ObservableInterpretable struct { - Interpretable + InterpretableV2 observers []StatefulObserver } // ID implements the Interpretable method to get the expression id associated with the step. func (oi *ObservableInterpretable) ID() int64 { - return oi.Interpretable.ID() + return oi.InterpretableV2.ID() +} + +// Exec implements the InterpretableV2 interface method. +func (oi *ObservableInterpretable) Exec(frame *ExecutionFrame) ref.Val { + return oi.ObserveExec(frame, func(any) {}) } // Eval proxies to the ObserveEval method while invoking a no-op callback to report the observations. func (oi *ObservableInterpretable) Eval(vars Activation) ref.Val { - return oi.ObserveEval(vars, func(any) {}) + return oi.ObserveExec(AsFrame(vars), func(any) {}) } // ObserveEval evaluates an interpretable and performs per-evaluation state-tracking. @@ -131,25 +169,65 @@ func (oi *ObservableInterpretable) Eval(vars Activation) ref.Val { // This method is concurrency safe and the expectation is that the observer function will use // a switch statement to determine the type of the state which has been reported back from the call. func (oi *ObservableInterpretable) ObserveEval(vars Activation, observer func(any)) ref.Val { - var err error + return oi.ObserveExec(AsFrame(vars), observer) +} + +// ObserveExec evaluates an interpretable and performs per-evaluation state-tracking. +// +// This method is concurrency safe and the expectation is that the observer function will use +// a switch statement to determine the type of the state which has been reported back from the call. +func (oi *ObservableInterpretable) ObserveExec(frame *ExecutionFrame, observer func(any)) ref.Val { // Initialize the state needed for the observers to function. for _, obs := range oi.observers { - vars, err = obs.InitState(vars) + state, err := obs.InitState(frame) if err != nil { return types.WrapErr(err) } // Provide an initial reference to the state to ensure state is available // even in cases of interrupting errors generated during evaluation. - observer(obs.GetState(vars)) + observer(state) } - result := oi.Interpretable.Eval(vars) + result := oi.InterpretableV2.Exec(frame) // Get the state which needs to be reported back as having been observed. for _, obs := range oi.observers { - observer(obs.GetState(vars)) + observer(obs.GetState(frame)) } return result } +// AsFrame promotes an Activation to an ExecutionFrame. +func AsFrame(a Activation) *ExecutionFrame { + if f, ok := a.(*ExecutionFrame); ok { + return f + } + frame := &ExecutionFrame{Activation: a} + // Walk the activation hierarchy to find a parent ExecutionFrame and inherit + // its shared context. + if parent := findFrame(a); parent != nil { + frame.ctx = parent.ctx + } + return frame +} + +// findFrame walks the activation hierarchy via Unwrap and Parent to locate an +// existing ExecutionFrame, if one exists. +func findFrame(a Activation) *ExecutionFrame { + if wrapper, ok := a.(activationWrapper); ok { + unwrapped := wrapper.Unwrap() + if f, ok := unwrapped.(*ExecutionFrame); ok { + return f + } + return findFrame(unwrapped) + } + if p := a.Parent(); p != nil { + if f, ok := p.(*ExecutionFrame); ok { + return f + } + return findFrame(p) + } + return nil +} + // Core Interpretable implementations used during the program planning phase. type evalTestOnly struct { @@ -162,9 +240,9 @@ func (test *evalTestOnly) ID() int64 { return test.id } -// Eval implements the Interpretable interface method. -func (test *evalTestOnly) Eval(ctx Activation) ref.Val { - val, err := test.Resolve(ctx) +// Exec implements the InterpretableV2 interface method. +func (test *evalTestOnly) Exec(frame *ExecutionFrame) ref.Val { + val, err := test.Resolve(frame) // Return an error if the resolve step fails if err != nil { return types.LabelErrNode(test.id, types.WrapErr(err)) @@ -175,6 +253,11 @@ func (test *evalTestOnly) Eval(ctx Activation) ref.Val { return test.Adapter().NativeToValue(val) } +// Eval implements the Interpretable interface method. +func (test *evalTestOnly) Eval(ctx Activation) ref.Val { + return test.Exec(AsFrame(ctx)) +} + // AddQualifier appends a qualifier that will always and only perform a presence test. func (test *evalTestOnly) AddQualifier(q Qualifier) (Attribute, error) { cq, ok := q.(ConstantQualifier) @@ -194,7 +277,7 @@ func (q *testOnlyQualifier) Qualify(vars Activation, obj any) (any, error) { if err != nil { return nil, err } - if unk, isUnk := out.(types.Unknown); isUnk { + if unk, isUnk := out.(*types.Unknown); isUnk { return unk, nil } return present, nil @@ -230,6 +313,11 @@ func (cons *evalConst) ID() int64 { return cons.id } +// Exec implements the InterpretableV2 interface method. +func (cons *evalConst) Exec(frame *ExecutionFrame) ref.Val { + return cons.val +} + // Eval implements the Interpretable interface method. func (cons *evalConst) Eval(ctx Activation) ref.Val { return cons.val @@ -242,7 +330,7 @@ func (cons *evalConst) Value() ref.Val { type evalOr struct { id int64 - terms []Interpretable + terms []InterpretableV2 } // ID implements the Interpretable interface method. @@ -250,12 +338,12 @@ func (or *evalOr) ID() int64 { return or.id } -// Eval implements the Interpretable interface method. -func (or *evalOr) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (or *evalOr) Exec(frame *ExecutionFrame) ref.Val { var err ref.Val = nil var unk *types.Unknown for _, term := range or.terms { - val := term.Eval(ctx) + val := term.Exec(frame) boolVal, ok := val.(types.Bool) // short-circuit on true. if ok && boolVal == types.True { @@ -283,9 +371,14 @@ func (or *evalOr) Eval(ctx Activation) ref.Val { return types.False } +// Eval implements the Interpretable interface method. +func (or *evalOr) Eval(ctx Activation) ref.Val { + return or.Exec(AsFrame(ctx)) +} + type evalAnd struct { id int64 - terms []Interpretable + terms []InterpretableV2 } // ID implements the Interpretable interface method. @@ -293,12 +386,12 @@ func (and *evalAnd) ID() int64 { return and.id } -// Eval implements the Interpretable interface method. -func (and *evalAnd) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (and *evalAnd) Exec(frame *ExecutionFrame) ref.Val { var err ref.Val = nil var unk *types.Unknown for _, term := range and.terms { - val := term.Eval(ctx) + val := term.Exec(frame) boolVal, ok := val.(types.Bool) // short-circuit on false. if ok && boolVal == types.False { @@ -326,10 +419,15 @@ func (and *evalAnd) Eval(ctx Activation) ref.Val { return types.True } +// Eval implements the Interpretable interface method. +func (and *evalAnd) Eval(ctx Activation) ref.Val { + return and.Exec(AsFrame(ctx)) +} + type evalEq struct { id int64 - lhs Interpretable - rhs Interpretable + lhs InterpretableV2 + rhs InterpretableV2 } // ID implements the Interpretable interface method. @@ -337,19 +435,30 @@ func (eq *evalEq) ID() int64 { return eq.id } -// Eval implements the Interpretable interface method. -func (eq *evalEq) Eval(ctx Activation) ref.Val { - lVal := eq.lhs.Eval(ctx) - rVal := eq.rhs.Eval(ctx) - if types.IsUnknownOrError(lVal) { +// Exec implements the InterpretableV2 interface method. +func (eq *evalEq) Exec(frame *ExecutionFrame) ref.Val { + lVal := eq.lhs.Exec(frame) + if types.IsError(lVal) { return lVal } - if types.IsUnknownOrError(rVal) { + rVal := eq.rhs.Exec(frame) + if types.IsError(rVal) { return rVal } + var unk *types.Unknown + unk, _ = types.MaybeMergeUnknowns(lVal, unk) + unk, _ = types.MaybeMergeUnknowns(rVal, unk) + if unk != nil { + return unk + } return types.Equal(lVal, rVal) } +// Eval implements the Interpretable interface method. +func (eq *evalEq) Eval(ctx Activation) ref.Val { + return eq.Exec(AsFrame(ctx)) +} + // Function implements the InterpretableCall interface method. func (*evalEq) Function() string { return operators.Equals @@ -361,14 +470,14 @@ func (*evalEq) OverloadID() string { } // Args implements the InterpretableCall interface method. -func (eq *evalEq) Args() []Interpretable { - return []Interpretable{eq.lhs, eq.rhs} +func (eq *evalEq) Args() []InterpretableV2 { + return []InterpretableV2{eq.lhs, eq.rhs} } type evalNe struct { id int64 - lhs Interpretable - rhs Interpretable + lhs InterpretableV2 + rhs InterpretableV2 } // ID implements the Interpretable interface method. @@ -376,19 +485,30 @@ func (ne *evalNe) ID() int64 { return ne.id } -// Eval implements the Interpretable interface method. -func (ne *evalNe) Eval(ctx Activation) ref.Val { - lVal := ne.lhs.Eval(ctx) - rVal := ne.rhs.Eval(ctx) - if types.IsUnknownOrError(lVal) { +// Exec implements the InterpretableV2 interface method. +func (ne *evalNe) Exec(frame *ExecutionFrame) ref.Val { + lVal := ne.lhs.Exec(frame) + if types.IsError(lVal) { return lVal } - if types.IsUnknownOrError(rVal) { + rVal := ne.rhs.Exec(frame) + if types.IsError(rVal) { return rVal } + var unk *types.Unknown + unk, _ = types.MaybeMergeUnknowns(lVal, unk) + unk, _ = types.MaybeMergeUnknowns(rVal, unk) + if unk != nil { + return unk + } return types.Bool(types.Equal(lVal, rVal) != types.True) } +// Eval implements the Interpretable interface method. +func (ne *evalNe) Eval(ctx Activation) ref.Val { + return ne.Exec(AsFrame(ctx)) +} + // Function implements the InterpretableCall interface method. func (*evalNe) Function() string { return operators.NotEquals @@ -400,8 +520,8 @@ func (*evalNe) OverloadID() string { } // Args implements the InterpretableCall interface method. -func (ne *evalNe) Args() []Interpretable { - return []Interpretable{ne.lhs, ne.rhs} +func (ne *evalNe) Args() []InterpretableV2 { + return []InterpretableV2{ne.lhs, ne.rhs} } type evalZeroArity struct { @@ -416,9 +536,14 @@ func (zero *evalZeroArity) ID() int64 { return zero.id } +// Exec implements the InterpretableV2 interface method. +func (zero *evalZeroArity) Exec(frame *ExecutionFrame) ref.Val { + return types.LabelErrNode(zero.id, zero.impl()) +} + // Eval implements the Interpretable interface method. func (zero *evalZeroArity) Eval(ctx Activation) ref.Val { - return types.LabelErrNode(zero.id, zero.impl()) + return zero.Exec(AsFrame(ctx)) } // Function implements the InterpretableCall interface method. @@ -432,15 +557,15 @@ func (zero *evalZeroArity) OverloadID() string { } // Args returns the argument to the unary function. -func (zero *evalZeroArity) Args() []Interpretable { - return []Interpretable{} +func (zero *evalZeroArity) Args() []InterpretableV2 { + return []InterpretableV2{} } type evalUnary struct { id int64 function string overload string - arg Interpretable + arg InterpretableV2 trait int impl functions.UnaryOp nonStrict bool @@ -451,9 +576,9 @@ func (un *evalUnary) ID() int64 { return un.id } -// Eval implements the Interpretable interface method. -func (un *evalUnary) Eval(ctx Activation) ref.Val { - argVal := un.arg.Eval(ctx) +// Exec implements the InterpretableV2 interface method. +func (un *evalUnary) Exec(frame *ExecutionFrame) ref.Val { + argVal := un.arg.Exec(frame) // Early return if the argument to the function is unknown or error. strict := !un.nonStrict if strict && types.IsUnknownOrError(argVal) { @@ -472,6 +597,11 @@ func (un *evalUnary) Eval(ctx Activation) ref.Val { return types.NewErrWithNodeID(un.id, "no such overload: %s", un.function) } +// Eval implements the Interpretable interface method. +func (un *evalUnary) Eval(ctx Activation) ref.Val { + return un.Exec(AsFrame(ctx)) +} + // Function implements the InterpretableCall interface method. func (un *evalUnary) Function() string { return un.function @@ -483,16 +613,16 @@ func (un *evalUnary) OverloadID() string { } // Args returns the argument to the unary function. -func (un *evalUnary) Args() []Interpretable { - return []Interpretable{un.arg} +func (un *evalUnary) Args() []InterpretableV2 { + return []InterpretableV2{un.arg} } type evalBinary struct { id int64 function string overload string - lhs Interpretable - rhs Interpretable + lhs InterpretableV2 + rhs InterpretableV2 trait int impl functions.BinaryOp nonStrict bool @@ -503,18 +633,23 @@ func (bin *evalBinary) ID() int64 { return bin.id } -// Eval implements the Interpretable interface method. -func (bin *evalBinary) Eval(ctx Activation) ref.Val { - lVal := bin.lhs.Eval(ctx) - rVal := bin.rhs.Eval(ctx) - // Early return if any argument to the function is unknown or error. +// Exec implements the InterpretableV2 interface method. +func (bin *evalBinary) Exec(frame *ExecutionFrame) ref.Val { + lVal := bin.lhs.Exec(frame) strict := !bin.nonStrict + if strict && types.IsError(lVal) { + return lVal + } + rVal := bin.rhs.Exec(frame) + if strict && types.IsError(rVal) { + return rVal + } if strict { - if types.IsUnknownOrError(lVal) { - return lVal - } - if types.IsUnknownOrError(rVal) { - return rVal + var unk *types.Unknown + unk, _ = types.MaybeMergeUnknowns(lVal, unk) + unk, _ = types.MaybeMergeUnknowns(rVal, unk) + if unk != nil { + return unk } } // If the implementation is bound and the argument value has the right traits required to @@ -530,6 +665,11 @@ func (bin *evalBinary) Eval(ctx Activation) ref.Val { return types.NewErrWithNodeID(bin.id, "no such overload: %s", bin.function) } +// Eval implements the Interpretable interface method. +func (bin *evalBinary) Eval(ctx Activation) ref.Val { + return bin.Exec(AsFrame(ctx)) +} + // Function implements the InterpretableCall interface method. func (bin *evalBinary) Function() string { return bin.function @@ -541,22 +681,22 @@ func (bin *evalBinary) OverloadID() string { } // Args returns the argument to the unary function. -func (bin *evalBinary) Args() []Interpretable { - return []Interpretable{bin.lhs, bin.rhs} +func (bin *evalBinary) Args() []InterpretableV2 { + return []InterpretableV2{bin.lhs, bin.rhs} } type evalVarArgs struct { id int64 function string overload string - args []Interpretable + args []InterpretableV2 trait int impl functions.FunctionOp nonStrict bool } // NewCall creates a new call Interpretable. -func NewCall(id int64, function, overload string, args []Interpretable, impl functions.FunctionOp) InterpretableCall { +func NewCall(id int64, function, overload string, args []InterpretableV2, impl functions.FunctionOp) InterpretableCall { return &evalVarArgs{ id: id, function: function, @@ -571,17 +711,23 @@ func (fn *evalVarArgs) ID() int64 { return fn.id } -// Eval implements the Interpretable interface method. -func (fn *evalVarArgs) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (fn *evalVarArgs) Exec(frame *ExecutionFrame) ref.Val { argVals := make([]ref.Val, len(fn.args)) - // Early return if any argument to the function is unknown or error. strict := !fn.nonStrict + var unk *types.Unknown for i, arg := range fn.args { - argVals[i] = arg.Eval(ctx) - if strict && types.IsUnknownOrError(argVals[i]) { - return argVals[i] + argVals[i] = arg.Exec(frame) + if strict { + if types.IsError(argVals[i]) { + return argVals[i] + } + unk, _ = types.MaybeMergeUnknowns(argVals[i], unk) } } + if strict && unk != nil { + return unk + } // If the implementation is bound and the argument value has the right traits required to // invoke it, then call the implementation. arg0 := argVals[0] @@ -596,6 +742,11 @@ func (fn *evalVarArgs) Eval(ctx Activation) ref.Val { return types.NewErrWithNodeID(fn.id, "no such overload: %s %d", fn.function, fn.id) } +// Eval implements the Interpretable interface method. +func (fn *evalVarArgs) Eval(ctx Activation) ref.Val { + return fn.Exec(AsFrame(ctx)) +} + // Function implements the InterpretableCall interface method. func (fn *evalVarArgs) Function() string { return fn.function @@ -607,13 +758,13 @@ func (fn *evalVarArgs) OverloadID() string { } // Args returns the argument to the unary function. -func (fn *evalVarArgs) Args() []Interpretable { +func (fn *evalVarArgs) Args() []InterpretableV2 { return fn.args } type evalList struct { id int64 - elems []Interpretable + elems []InterpretableV2 optionals []bool hasOptionals bool adapter types.Adapter @@ -624,31 +775,44 @@ func (l *evalList) ID() int64 { return l.id } -// Eval implements the Interpretable interface method. -func (l *evalList) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (l *evalList) Exec(frame *ExecutionFrame) ref.Val { elemVals := make([]ref.Val, 0, len(l.elems)) - // If any argument is unknown or error early terminate. + var unk *types.Unknown for i, elem := range l.elems { - elemVal := elem.Eval(ctx) - if types.IsUnknownOrError(elemVal) { + elemVal := elem.Exec(frame) + if types.IsError(elemVal) { return elemVal } + unk, _ = types.MaybeMergeUnknowns(elemVal, unk) if l.hasOptionals && l.optionals[i] { - optVal, ok := elemVal.(*types.Optional) - if !ok { - return types.LabelErrNode(l.id, invalidOptionalElementInit(elemVal)) - } - if !optVal.HasValue() { - continue + if types.IsUnknown(elemVal) { + // skip optional checks for unknown values as they aren't fully resolved yet. + } else { + optVal, ok := elemVal.(*types.Optional) + if !ok { + return types.LabelErrNode(l.id, invalidOptionalElementInit(elemVal)) + } + if !optVal.HasValue() { + continue + } + elemVal = optVal.GetValue() } - elemVal = optVal.GetValue() } elemVals = append(elemVals, elemVal) } - return l.adapter.NativeToValue(elemVals) + if unk != nil { + return unk + } + return types.NewRefValList(l.adapter, elemVals) } -func (l *evalList) InitVals() []Interpretable { +// Eval implements the Interpretable interface method. +func (l *evalList) Eval(ctx Activation) ref.Val { + return l.Exec(AsFrame(ctx)) +} + +func (l *evalList) InitVals() []InterpretableV2 { return l.elems } @@ -658,8 +822,8 @@ func (l *evalList) Type() ref.Type { type evalMap struct { id int64 - keys []Interpretable - vals []Interpretable + keys []InterpretableV2 + vals []InterpretableV2 optionals []bool hasOptionals bool adapter types.Adapter @@ -670,20 +834,24 @@ func (m *evalMap) ID() int64 { return m.id } -// Eval implements the Interpretable interface method. -func (m *evalMap) Eval(ctx Activation) ref.Val { - entries := make(map[ref.Val]ref.Val) - // If any argument is unknown or error early terminate. +// Exec implements the InterpretableV2 interface method. +func (m *evalMap) Exec(frame *ExecutionFrame) ref.Val { + entries := make(map[ref.Val]ref.Val, len(m.keys)) + var unk *types.Unknown for i, key := range m.keys { - keyVal := key.Eval(ctx) - if types.IsUnknownOrError(keyVal) { + keyVal := key.Exec(frame) + if types.IsError(keyVal) { return keyVal } - valVal := m.vals[i].Eval(ctx) - if types.IsUnknownOrError(valVal) { + unk, _ = types.MaybeMergeUnknowns(keyVal, unk) + + valVal := m.vals[i].Exec(frame) + if types.IsError(valVal) { return valVal } - if m.hasOptionals && m.optionals[i] { + unk, _ = types.MaybeMergeUnknowns(valVal, unk) + + if m.hasOptionals && m.optionals[i] && !types.IsUnknown(valVal) { optVal, ok := valVal.(*types.Optional) if !ok { return types.LabelErrNode(m.id, invalidOptionalEntryInit(keyVal, valVal)) @@ -696,14 +864,22 @@ func (m *evalMap) Eval(ctx Activation) ref.Val { } entries[keyVal] = valVal } - return m.adapter.NativeToValue(entries) + if unk != nil { + return unk + } + return types.NewRefValMap(m.adapter, entries) } -func (m *evalMap) InitVals() []Interpretable { +// Eval implements the Interpretable interface method. +func (m *evalMap) Eval(ctx Activation) ref.Val { + return m.Exec(AsFrame(ctx)) +} + +func (m *evalMap) InitVals() []InterpretableV2 { if len(m.keys) != len(m.vals) { return nil } - result := make([]Interpretable, len(m.keys)+len(m.vals)) + result := make([]InterpretableV2, len(m.keys)+len(m.vals)) idx := 0 for i, k := range m.keys { v := m.vals[i] @@ -723,7 +899,7 @@ type evalObj struct { id int64 typeName string fields []string - vals []Interpretable + vals []InterpretableV2 optionals []bool hasOptionals bool provider types.Provider @@ -734,16 +910,17 @@ func (o *evalObj) ID() int64 { return o.id } -// Eval implements the Interpretable interface method. -func (o *evalObj) Eval(ctx Activation) ref.Val { - fieldVals := make(map[string]ref.Val) - // If any argument is unknown or error early terminate. +// Exec implements the InterpretableV2 interface method. +func (o *evalObj) Exec(frame *ExecutionFrame) ref.Val { + fieldVals := make(map[string]ref.Val, len(o.fields)) + var unk *types.Unknown for i, field := range o.fields { - val := o.vals[i].Eval(ctx) - if types.IsUnknownOrError(val) { + val := o.vals[i].Exec(frame) + if types.IsError(val) { return val } - if o.hasOptionals && o.optionals[i] { + unk, _ = types.MaybeMergeUnknowns(val, unk) + if o.hasOptionals && o.optionals[i] && !types.IsUnknown(val) { optVal, ok := val.(*types.Optional) if !ok { return types.LabelErrNode(o.id, invalidOptionalEntryInit(field, val)) @@ -756,11 +933,19 @@ func (o *evalObj) Eval(ctx Activation) ref.Val { } fieldVals[field] = val } + if unk != nil { + return unk + } return types.LabelErrNode(o.id, o.provider.NewValue(o.typeName, fieldVals)) } +// Eval implements the Interpretable interface method. +func (o *evalObj) Eval(ctx Activation) ref.Val { + return o.Exec(AsFrame(ctx)) +} + // InitVals implements the InterpretableConstructor interface method. -func (o *evalObj) InitVals() []Interpretable { +func (o *evalObj) InitVals() []InterpretableV2 { return o.vals } @@ -774,11 +959,11 @@ type evalFold struct { accuVar string iterVar string iterVar2 string - iterRange Interpretable - accu Interpretable - cond Interpretable - step Interpretable - result Interpretable + iterRange InterpretableV2 + accu InterpretableV2 + cond InterpretableV2 + step InterpretableV2 + result InterpretableV2 adapter types.Adapter // note an exhaustive fold will ensure that all branches are evaluated @@ -793,13 +978,13 @@ func (fold *evalFold) ID() int64 { return fold.id } -// Eval implements the Interpretable interface method. -func (fold *evalFold) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (fold *evalFold) Exec(frame *ExecutionFrame) ref.Val { // Initialize the folder interface - f := newFolder(fold, ctx) + f := newFolder(fold, frame) defer releaseFolder(f) - foldRange := fold.iterRange.Eval(ctx) + foldRange := fold.iterRange.Exec(frame) if types.IsUnknownOrError(foldRange) { return foldRange } @@ -824,14 +1009,19 @@ func (fold *evalFold) Eval(ctx Activation) ref.Val { return f.foldIterable(iterable) } +// Eval implements the Interpretable interface method. +func (fold *evalFold) Eval(ctx Activation) ref.Val { + return fold.Exec(AsFrame(ctx)) +} + // Optional Interpretable implementations that specialize, subsume, or extend the core evaluation // plan via decorators. // evalSetMembership is an Interpretable implementation which tests whether an input value // exists within the set of map keys used to model a set. type evalSetMembership struct { - inst Interpretable - arg Interpretable + inst InterpretableV2 + arg InterpretableV2 valueSet map[ref.Val]ref.Val } @@ -840,9 +1030,9 @@ func (e *evalSetMembership) ID() int64 { return e.inst.ID() } -// Eval implements the Interpretable interface method. -func (e *evalSetMembership) Eval(ctx Activation) ref.Val { - val := e.arg.Eval(ctx) +// Exec implements the InterpretableV2 interface method. +func (e *evalSetMembership) Exec(frame *ExecutionFrame) ref.Val { + val := e.arg.Exec(frame) if types.IsUnknownOrError(val) { return val } @@ -852,18 +1042,28 @@ func (e *evalSetMembership) Eval(ctx Activation) ref.Val { return types.False } +// Eval implements the Interpretable interface method. +func (e *evalSetMembership) Eval(ctx Activation) ref.Val { + return e.Exec(AsFrame(ctx)) +} + // evalWatch is an Interpretable implementation that wraps the execution of a given // expression so that it may observe the computed value and send it to an observer. type evalWatch struct { - Interpretable + InterpretableV2 observer EvalObserver } +// Exec implements the InterpretableV2 interface method. +func (e *evalWatch) Exec(frame *ExecutionFrame) ref.Val { + val := e.InterpretableV2.Exec(frame) + e.observer(frame, e.ID(), e.InterpretableV2, val) + return val +} + // Eval implements the Interpretable interface method. func (e *evalWatch) Eval(vars Activation) ref.Val { - val := e.Interpretable.Eval(vars) - e.observer(vars, e.ID(), e.Interpretable, val) - return val + return e.Exec(AsFrame(vars)) } // evalWatchAttr describes a watcher of an InterpretableAttribute Interpretable. @@ -918,11 +1118,16 @@ func (e *evalWatchAttr) AddQualifier(q Qualifier) (Attribute, error) { return e, err } +// Exec implements the InterpretableV2 interface method. +func (e *evalWatchAttr) Exec(frame *ExecutionFrame) ref.Val { + val := e.InterpretableAttribute.Exec(frame) + e.observer(frame, e.ID(), e.InterpretableAttribute, val) + return val +} + // Eval implements the Interpretable interface method. func (e *evalWatchAttr) Eval(vars Activation) ref.Val { - val := e.InterpretableAttribute.Eval(vars) - e.observer(vars, e.ID(), e.InterpretableAttribute, val) - return val + return e.Exec(AsFrame(vars)) } // evalWatchConstQual observes the qualification of an object using a constant boolean, int, @@ -1049,17 +1254,22 @@ type evalWatchConst struct { observer EvalObserver } -// Eval implements the Interpretable interface method. -func (e *evalWatchConst) Eval(vars Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (e *evalWatchConst) Exec(frame *ExecutionFrame) ref.Val { val := e.Value() - e.observer(vars, e.ID(), e.InterpretableConst, val) + e.observer(frame, e.ID(), e.InterpretableConst, val) return val } +// Eval implements the Interpretable interface method. +func (e *evalWatchConst) Eval(vars Activation) ref.Val { + return e.Exec(AsFrame(vars)) +} + // evalExhaustiveOr is just like evalOr, but does not short-circuit argument evaluation. type evalExhaustiveOr struct { id int64 - terms []Interpretable + terms []InterpretableV2 } // ID implements the Interpretable interface method. @@ -1067,13 +1277,13 @@ func (or *evalExhaustiveOr) ID() int64 { return or.id } -// Eval implements the Interpretable interface method. -func (or *evalExhaustiveOr) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (or *evalExhaustiveOr) Exec(frame *ExecutionFrame) ref.Val { var err ref.Val = nil var unk *types.Unknown isTrue := false for _, term := range or.terms { - val := term.Eval(ctx) + val := term.Exec(frame) boolVal, ok := val.(types.Bool) // flag the result as true if ok && boolVal == types.True { @@ -1103,10 +1313,15 @@ func (or *evalExhaustiveOr) Eval(ctx Activation) ref.Val { return types.False } +// Eval implements the Interpretable interface method. +func (or *evalExhaustiveOr) Eval(ctx Activation) ref.Val { + return or.Exec(AsFrame(ctx)) +} + // evalExhaustiveAnd is just like evalAnd, but does not short-circuit argument evaluation. type evalExhaustiveAnd struct { id int64 - terms []Interpretable + terms []InterpretableV2 } // ID implements the Interpretable interface method. @@ -1114,13 +1329,13 @@ func (and *evalExhaustiveAnd) ID() int64 { return and.id } -// Eval implements the Interpretable interface method. -func (and *evalExhaustiveAnd) Eval(ctx Activation) ref.Val { +// Exec implements the InterpretableV2 interface method. +func (and *evalExhaustiveAnd) Exec(frame *ExecutionFrame) ref.Val { var err ref.Val = nil var unk *types.Unknown isFalse := false for _, term := range and.terms { - val := term.Eval(ctx) + val := term.Exec(frame) boolVal, ok := val.(types.Bool) // short-circuit on false. if ok && boolVal == types.False { @@ -1150,6 +1365,11 @@ func (and *evalExhaustiveAnd) Eval(ctx Activation) ref.Val { return types.True } +// Eval implements the Interpretable interface method. +func (and *evalExhaustiveAnd) Eval(ctx Activation) ref.Val { + return and.Exec(AsFrame(ctx)) +} + // evalExhaustiveConditional is like evalConditional, but does not short-circuit argument // evaluation. type evalExhaustiveConditional struct { @@ -1163,11 +1383,11 @@ func (cond *evalExhaustiveConditional) ID() int64 { return cond.id } -// Eval implements the Interpretable interface method. -func (cond *evalExhaustiveConditional) Eval(ctx Activation) ref.Val { - cVal := cond.attr.expr.Eval(ctx) - tVal, tErr := cond.attr.truthy.Resolve(ctx) - fVal, fErr := cond.attr.falsy.Resolve(ctx) +// Exec implements the InterpretableV2 interface method. +func (cond *evalExhaustiveConditional) Exec(frame *ExecutionFrame) ref.Val { + cVal := cond.attr.expr.Exec(frame) + tVal, tErr := cond.attr.truthy.Resolve(frame) + fVal, fErr := cond.attr.falsy.Resolve(frame) cBool, ok := cVal.(types.Bool) if !ok { return types.ValOrErr(cVal, "no such overload") @@ -1184,6 +1404,11 @@ func (cond *evalExhaustiveConditional) Eval(ctx Activation) ref.Val { return cond.adapter.NativeToValue(fVal) } +// Eval implements the Interpretable interface method. +func (cond *evalExhaustiveConditional) Eval(ctx Activation) ref.Val { + return cond.Exec(AsFrame(ctx)) +} + // evalAttr evaluates an Attribute value. type evalAttr struct { adapter types.Adapter @@ -1215,15 +1440,20 @@ func (a *evalAttr) Adapter() types.Adapter { return a.adapter } -// Eval implements the Interpretable interface method. -func (a *evalAttr) Eval(ctx Activation) ref.Val { - v, err := a.attr.Resolve(ctx) +// Exec implements the InterpretableV2 interface method. +func (a *evalAttr) Exec(frame *ExecutionFrame) ref.Val { + v, err := a.attr.Resolve(frame) if err != nil { return types.LabelErrNode(a.ID(), types.WrapErr(err)) } return a.adapter.NativeToValue(v) } +// Eval implements the Interpretable interface method. +func (a *evalAttr) Eval(ctx Activation) ref.Val { + return a.Exec(AsFrame(ctx)) +} + // Qualify proxies to the Attribute's Qualify method. func (a *evalAttr) Qualify(vars Activation, obj any) (any, error) { return a.attr.Qualify(vars, obj) @@ -1249,7 +1479,7 @@ type evalWatchConstructor struct { } // InitVals implements the InterpretableConstructor InitVals function. -func (c *evalWatchConstructor) InitVals() []Interpretable { +func (c *evalWatchConstructor) InitVals() []InterpretableV2 { return c.constructor.InitVals() } @@ -1263,11 +1493,16 @@ func (c *evalWatchConstructor) ID() int64 { return c.constructor.ID() } +// Exec implements the InterpretableV2 interface method. +func (c *evalWatchConstructor) Exec(frame *ExecutionFrame) ref.Val { + val := c.constructor.Exec(frame) + c.observer(frame, c.ID(), c.constructor, val) + return val +} + // Eval implements the Interpretable Eval function. func (c *evalWatchConstructor) Eval(vars Activation) ref.Val { - val := c.constructor.Eval(vars) - c.observer(vars, c.ID(), c.constructor, val) - return val + return c.Exec(AsFrame(vars)) } func invalidOptionalEntryInit(field any, value ref.Val) ref.Val { @@ -1279,10 +1514,10 @@ func invalidOptionalElementInit(value ref.Val) ref.Val { } // newFolder creates or initializes a pooled folder instance. -func newFolder(eval *evalFold, ctx Activation) *folder { +func newFolder(eval *evalFold, frame *ExecutionFrame) *folder { f := folderPool.Get().(*folder) f.evalFold = eval - f.activation = ctx + f.frame = frame.Push(f) return f } @@ -1303,7 +1538,7 @@ func releaseFolder(f *folder) { // cel.bind or cel.@block. type folder struct { *evalFold - activation Activation + frame *ExecutionFrame // fold state objects. accuVal ref.Val @@ -1322,16 +1557,16 @@ func (f *folder) foldIterable(iterable traits.Iterable) ref.Val { for it.HasNext() == types.True { f.iterVar1Val = it.Next() - cond := f.cond.Eval(f) + cond := f.cond.Exec(f.frame) condBool, ok := cond.(types.Bool) if f.interrupted || (!f.exhaustive && ok && condBool != types.True) { return f.evalResult() } // Update the accumulation value and check for eval interuption. - f.accuVal = f.step.Eval(f) + f.accuVal = f.step.Exec(f.frame) f.initialized = true - if f.interruptable && checkInterrupt(f.activation) { + if f.interruptable && f.frame.CheckInterrupt() { f.interrupted = true return f.evalResult() } @@ -1348,16 +1583,16 @@ func (f *folder) FoldEntry(key, val any) bool { // Terminate evaluation if evaluation is interrupted or the condition is not true and exhaustive // eval is not enabled. - cond := f.cond.Eval(f) + cond := f.cond.Exec(f.frame) condBool, ok := cond.(types.Bool) if f.interrupted || (!f.exhaustive && ok && condBool != types.True) { return false } // Update the accumulation value and check for eval interuption. - f.accuVal = f.step.Eval(f) + f.accuVal = f.step.Exec(f.frame) f.initialized = true - if f.interruptable && checkInterrupt(f.activation) { + if f.interruptable && f.frame.CheckInterrupt() { f.interrupted = true return false } @@ -1371,7 +1606,7 @@ func (f *folder) ResolveName(name string) (any, bool) { if name == f.accuVar { if !f.initialized { f.initialized = true - initVal := f.accu.Eval(f.activation) + initVal := f.accu.Exec(f.frame.parent) if !f.exhaustive { if l, isList := initVal.(traits.Lister); isList && l.Size() == types.IntZero { initVal = types.NewMutableList(f.adapter) @@ -1396,23 +1631,43 @@ func (f *folder) ResolveName(name string) (any, bool) { return f.iterVar2Val, true } } - return f.activation.ResolveName(name) + return f.frame.parent.ResolveName(name) } // Parent returns the activation embedded into the folder. func (f *folder) Parent() Activation { - return f.activation + return f.frame.parent } // Unwrap returns the parent activation, thus omitting access to local state func (f *folder) Unwrap() Activation { - return f.activation + return f.frame.parent +} + +// IsLocalVariable reports whether the variable name is locally bound by the folder scope. +func (f *folder) IsLocalVariable(name string) bool { + if name == f.accuVar { + return true + } + if !f.computeResult && (name == f.iterVar || name == f.iterVar2) { + return true + } + parent := f.Parent() + if parent == nil { + return false + } + if varHolder, ok := parent.(localVariableHolder); ok { + if varHolder.IsLocalVariable(name) { + return true + } + } + return false } // UnknownAttributePatterns implements the PartialActivation interface returning the unknown patterns // if they were provided to the input activation, or an empty set if the proxied activation is not partial. func (f *folder) UnknownAttributePatterns() []*AttributePattern { - if pv, ok := f.activation.(partialActivationConverter); ok { + if pv, ok := f.frame.parent.Activation.(partialActivationConverter); ok { if partial, isPartial := pv.AsPartialActivation(); isPartial { return partial.UnknownAttributePatterns() } @@ -1421,7 +1676,7 @@ func (f *folder) UnknownAttributePatterns() []*AttributePattern { } func (f *folder) AsPartialActivation() (PartialActivation, bool) { - if pv, ok := f.activation.(partialActivationConverter); ok { + if pv, ok := f.frame.parent.Activation.(partialActivationConverter); ok { if _, isPartial := pv.AsPartialActivation(); isPartial { return f, true } @@ -1435,7 +1690,7 @@ func (f *folder) evalResult() ref.Val { if f.interrupted { return types.WrapErr(InterruptError{}) } - res := f.result.Eval(f) + res := f.result.Exec(f.frame) // Convert a mutable list or map to an immutable one if the comprehension has generated a list or // map as a result. if !types.IsUnknownOrError(res) && f.mutableValue { @@ -1452,7 +1707,8 @@ func (f *folder) evalResult() ref.Val { // reset clears any state associated with folder evaluation. func (f *folder) reset() { f.evalFold = nil - f.activation = nil + f.frame.Pop() + f.frame = nil f.accuVal = nil f.iterVar1Val = nil f.iterVar2Val = nil @@ -1463,11 +1719,6 @@ func (f *folder) reset() { f.computeResult = false } -func checkInterrupt(a Activation) bool { - stop, found := a.ResolveName("#interrupted") - return found && stop == true -} - // InterruptError is a specialized error type used to signal that program evaluation should check // whether a context cancellation is responsible for the error. type InterruptError struct{} diff --git a/vendor/github.com/google/cel-go/interpreter/interpreter.go b/vendor/github.com/google/cel-go/interpreter/interpreter.go index d81ef1280..ef13ab922 100644 --- a/vendor/github.com/google/cel-go/interpreter/interpreter.go +++ b/vendor/github.com/google/cel-go/interpreter/interpreter.go @@ -29,11 +29,11 @@ import ( // PlannerOption configures the program plan options during interpretable setup. type PlannerOption func(*planner) (*planner, error) -// Interpreter generates a new Interpretable from a checked or unchecked expression. +// Interpreter generates a new InterpretableV2 from a checked or unchecked expression. type Interpreter interface { - // NewInterpretable creates an Interpretable from a checked expression and an + // NewInterpretable creates an InterpretableV2 from a checked expression and an // optional list of PlannerOption values. - NewInterpretable(exprAST *ast.AST, opts ...PlannerOption) (Interpretable, error) + NewInterpretable(exprAST *ast.AST, opts ...PlannerOption) (InterpretableV2, error) } // EvalObserver is a functional interface that accepts an expression id and an observed value. @@ -43,16 +43,16 @@ type EvalObserver func(vars Activation, id int64, programStep any, value ref.Val // StatefulObserver observes evaluation while tracking or utilizing stateful behavior. type StatefulObserver interface { - // InitState configures stateful metadata on the activation. - InitState(Activation) (Activation, error) + // InitState configures stateful metadata on the execution frame. + InitState(*ExecutionFrame) (any, error) - // GetState retrieves the stateful metadata from the activation. - GetState(Activation) any + // GetState retrieves the stateful metadata from the execution frame. + GetState(*ExecutionFrame) any // Observe passes the activation and relevant evaluation metadata to the observer. - // The observe method is expected to do the equivalent of GetState(vars) in order + // The observe method is expected to do the equivalent of GetState(AsFrame(activation)) // to find the metadata that needs to be updated upon invocation. - Observe(vars Activation, id int64, programStep any, value ref.Val) + Observe(Activation, int64, any, ref.Val) } // EvalCancelledError represents a cancelled program evaluation operation. @@ -106,37 +106,6 @@ func EvalStateObserver(opts ...evalStateOption) PlannerOption { } } -// evalStateConverter identifies an object which is convertible to an EvalState instance. -type evalStateConverter interface { - asEvalState() EvalState -} - -// evalStateActivation hides state in the Activation in a manner not accessible to expressions. -type evalStateActivation struct { - vars Activation - state EvalState -} - -// ResolveName proxies variable lookups to the backing activation. -func (esa evalStateActivation) ResolveName(name string) (any, bool) { - return esa.vars.ResolveName(name) -} - -// Parent proxies parent lookups to the backing activation. -func (esa evalStateActivation) Parent() Activation { - return esa.vars -} - -// AsPartialActivation supports conversion to a partial activation in order to detect unknown attributes. -func (esa evalStateActivation) AsPartialActivation() (PartialActivation, bool) { - return AsPartialActivation(esa.vars) -} - -// asEvalState implements the evalStateConverter method. -func (esa evalStateActivation) asEvalState() EvalState { - return esa.state -} - // activationWrapper identifies an object carrying local variables which should not be exposed to the user // Activations used for such purposes can be unwrapped to return the activation which omits local state. type activationWrapper interface { @@ -144,24 +113,10 @@ type activationWrapper interface { Unwrap() Activation } -// asEvalState walks the Activation hierarchy and returns the first EvalState found, if present. -func asEvalState(vars Activation) (EvalState, bool) { - if conv, ok := vars.(evalStateConverter); ok { - return conv.asEvalState(), true - } - // Check if the current activation wraps another activation. This is used to support - // wrappers such as the @block() activation which may be composed of a dynamicSlotActivation or a - // constantSlotActivation. In this case, the underlying activation is the portion which interacts - // with the EvalState. - if wrapper, ok := vars.(activationWrapper); ok { - unwrapped := wrapper.Unwrap() - // Recursively call asEvalState on the unwrapped activation. This will check the unwrapped value and its parents. - return asEvalState(unwrapped) - } - if vars.Parent() != nil { - return asEvalState(vars.Parent()) - } - return nil, false +// localVariableHolder identifies an Activation scope that holds local variables and supports testing +// whether a variable name is local to this scope. +type localVariableHolder interface { + IsLocalVariable(name string) bool } // evalStateFactory holds a reference to a factory function that produces an EvalState instance. @@ -169,32 +124,54 @@ type evalStateFactory struct { factory func() EvalState } -// InitState produces an EvalState instance and bundles it into the Activation in a way which is +// InitState produces an EvalState instance and bundles it into the ExecutionFrame in a way which is // not visible to expression evaluation. -func (et *evalStateFactory) InitState(vars Activation) (Activation, error) { +func (et *evalStateFactory) InitState(frame *ExecutionFrame) (any, error) { + if frame.ctx != nil && frame.ctx.state != nil { + return frame.ctx.state, nil + } state := et.factory() - return evalStateActivation{vars: vars, state: state}, nil + if frame.ctx == nil { + frame.ctx = evalContextPool.Get().(*evalContext) + } + frame.ctx.state = state + return state, nil } // GetState extracts the EvalState from the Activation. -func (et *evalStateFactory) GetState(vars Activation) any { - if state, found := asEvalState(vars); found { - return state +func (et *evalStateFactory) GetState(frame *ExecutionFrame) any { + if frame.ctx == nil { + return nil } - return nil + return frame.ctx.state } // Observe records the evaluation state for a given expression node and program step. func (et *evalStateFactory) Observe(vars Activation, id int64, programStep any, val ref.Val) { - state, found := asEvalState(vars) - if !found { + frame := AsFrame(vars) + if frame.ctx == nil || frame.ctx.state == nil { return } - state.SetValue(id, val) + frame.ctx.state.SetValue(id, val) } // CustomDecorator configures a custom interpretable decorator for the program. func CustomDecorator(dec InterpretableDecorator) PlannerOption { + return func(p *planner) (*planner, error) { + dec2 := func(i InterpretableV2) (InterpretableV2, error) { + legacy, err := dec(i) + if err != nil { + return nil, err + } + return adaptToV2(legacy), nil + } + p.decorators = append(p.decorators, dec2) + return p, nil + } +} + +// CustomDecoratorV2 configures a custom V2 interpretable decorator for the program. +func CustomDecoratorV2(dec InterpretableDecoratorV2) PlannerOption { return func(p *planner) (*planner, error) { p.decorators = append(p.decorators, dec) return p, nil @@ -207,7 +184,7 @@ func CustomDecorator(dec InterpretableDecorator) PlannerOption { // provided to the decorator. This decorator is not thread-safe, and the EvalState // must be reset between Eval() calls. func ExhaustiveEval() PlannerOption { - return CustomDecorator(decDisableShortcircuits()) + return CustomDecoratorV2(decDisableShortcircuits()) } // InterruptableEval annotates comprehension loops with information that indicates they @@ -216,13 +193,13 @@ func ExhaustiveEval() PlannerOption { // The custom activation is currently managed higher up in the stack within the 'cel' package // and should not require any custom support on behalf of callers. func InterruptableEval() PlannerOption { - return CustomDecorator(decInterruptFolds()) + return CustomDecoratorV2(decInterruptFolds()) } // Optimize will pre-compute operations such as list and map construction and optimize // call arguments to set membership tests. The set of optimizations will increase over time. func Optimize() PlannerOption { - return CustomDecorator(decOptimize()) + return CustomDecoratorV2(decOptimize()) } // RegexOptimization provides a way to replace an InterpretableCall for a regex function when the @@ -247,7 +224,7 @@ type RegexOptimization struct { // CompileRegexConstants compiles regex pattern string constants at program creation time and reports any regex pattern // compile errors. func CompileRegexConstants(regexOptimizations ...*RegexOptimization) PlannerOption { - return CustomDecorator(decRegexOptimizer(regexOptimizations...)) + return CustomDecoratorV2(decRegexOptimizer(regexOptimizations...)) } type exprInterpreter struct { @@ -273,10 +250,10 @@ func NewInterpreter(dispatcher Dispatcher, attrFactory: attrFactory} } -// NewIntepretable implements the Interpreter interface method. +// NewInterpretable implements the Interpreter interface method. func (i *exprInterpreter) NewInterpretable( checked *ast.AST, - opts ...PlannerOption) (Interpretable, error) { + opts ...PlannerOption) (InterpretableV2, error) { p := newPlanner(i.dispatcher, i.provider, i.adapter, i.attrFactory, i.container, checked) var err error for _, o := range opts { diff --git a/vendor/github.com/google/cel-go/interpreter/planner.go b/vendor/github.com/google/cel-go/interpreter/planner.go index 0bc38449c..396a9803f 100644 --- a/vendor/github.com/google/cel-go/interpreter/planner.go +++ b/vendor/github.com/google/cel-go/interpreter/planner.go @@ -43,7 +43,7 @@ func newPlanner(disp Dispatcher, container: cont, refMap: exprAST.ReferenceMap(), typeMap: exprAST.TypeMap(), - decorators: make([]InterpretableDecorator, 0), + decorators: make([]InterpretableDecoratorV2, 0), observers: make([]StatefulObserver, 0), } } @@ -57,7 +57,7 @@ type planner struct { container *containers.Container refMap map[int64]*ast.ReferenceInfo typeMap map[int64]*types.Type - decorators []InterpretableDecorator + decorators []InterpretableDecoratorV2 observers []StatefulObserver } @@ -72,7 +72,7 @@ type planBuilder struct { // useful for layering functionality into the evaluation that is not natively understood by CEL, // such as state-tracking, expression re-write, and possibly efficient thread-safe memoization of // repeated expressions. -func (p *planner) Plan(expr ast.Expr) (Interpretable, error) { +func (p *planner) Plan(expr ast.Expr) (InterpretableV2, error) { pb := &planBuilder{planner: p, localVars: make(map[string]int)} i, err := pb.plan(expr) if err != nil { @@ -81,10 +81,10 @@ func (p *planner) Plan(expr ast.Expr) (Interpretable, error) { if len(p.observers) == 0 { return i, nil } - return &ObservableInterpretable{Interpretable: i, observers: p.observers}, nil + return &ObservableInterpretable{InterpretableV2: i, observers: p.observers}, nil } -func (p *planBuilder) plan(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) plan(expr ast.Expr) (InterpretableV2, error) { switch expr.Kind() { case ast.CallKind: return p.decorate(p.planCall(expr)) @@ -109,7 +109,7 @@ func (p *planBuilder) plan(expr ast.Expr) (Interpretable, error) { // decorate applies the InterpretableDecorator functions to the given Interpretable. // Both the Interpretable and error generated by a Plan step are accepted as arguments // for convenience. -func (p *planBuilder) decorate(i Interpretable, err error) (Interpretable, error) { +func (p *planBuilder) decorate(i InterpretableV2, err error) (InterpretableV2, error) { if err != nil { return nil, err } @@ -123,7 +123,7 @@ func (p *planBuilder) decorate(i Interpretable, err error) (Interpretable, error } // planIdent creates an Interpretable that resolves an identifier from an Activation. -func (p *planBuilder) planIdent(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planIdent(expr ast.Expr) (InterpretableV2, error) { // Establish whether the identifier is in the reference map. if identRef, found := p.refMap[expr.ID()]; found { return p.planCheckedIdent(expr.ID(), identRef) @@ -142,7 +142,7 @@ func (p *planBuilder) planIdent(expr ast.Expr) (Interpretable, error) { }, nil } -func (p *planBuilder) planCheckedIdent(id int64, identRef *ast.ReferenceInfo) (Interpretable, error) { +func (p *planBuilder) planCheckedIdent(id int64, identRef *ast.ReferenceInfo) (InterpretableV2, error) { // Plan a constant reference if this is the case for this simple identifier. if identRef.Value != nil { return NewConstValue(id, identRef.Value), nil @@ -171,7 +171,7 @@ func (p *planBuilder) planCheckedIdent(id int64, identRef *ast.ReferenceInfo) (I // a) selects a field from a map or proto. // b) creates a field presence test for a select within a has() macro. // c) resolves the select expression to a namespaced identifier. -func (p *planBuilder) planSelect(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planSelect(expr ast.Expr) (InterpretableV2, error) { // If the Select id appears in the reference map from the CheckedExpr proto then it is either // a namespaced identifier or enum value. if identRef, found := p.refMap[expr.ID()]; found { @@ -227,7 +227,7 @@ func (p *planBuilder) planSelect(expr ast.Expr) (Interpretable, error) { // planCall creates a callable Interpretable while specializing for common functions and invocation // patterns. Specifically, conditional operators &&, ||, ?:, and (in)equality functions result in // optimized Interpretable values. -func (p *planBuilder) planCall(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planCall(expr ast.Expr) (InterpretableV2, error) { call := expr.AsCall() target, fnName, oName := p.resolveFunction(expr) argCount := len(call.Args()) @@ -237,7 +237,7 @@ func (p *planBuilder) planCall(expr ast.Expr) (Interpretable, error) { offset++ } - args := make([]Interpretable, argCount) + args := make([]InterpretableV2, argCount) if target != nil { arg, err := p.plan(target) if err != nil { @@ -281,6 +281,10 @@ func (p *planBuilder) planCall(expr ast.Expr) (Interpretable, error) { if fnDef == nil { fnDef, _ = p.disp.FindOverload(fnName) } + // Async overloads are planned into an evalAsyncFunc regardless of arity. + if fnDef != nil && fnDef.Async != nil { + return p.planCallAsync(expr, fnName, oName, fnDef, args) + } switch argCount { case 0: return p.planCallZero(expr, fnName, oName, fnDef) @@ -303,11 +307,29 @@ func (p *planBuilder) planCall(expr ast.Expr) (Interpretable, error) { } } +// planCallAsync generates an asynchronous callable Interpretable. +func (p *planBuilder) planCallAsync(expr ast.Expr, + function string, + overload string, + impl *functions.Overload, + args []InterpretableV2) (InterpretableV2, error) { + if impl == nil || impl.Async == nil { + return nil, fmt.Errorf("no such overload: %s()", function) + } + return &evalAsyncFunc{ + id: expr.ID(), + function: function, + overload: overload, + args: args, + impl: impl.Async, + }, nil +} + // planCallZero generates a zero-arity callable Interpretable. func (p *planBuilder) planCallZero(expr ast.Expr, function string, overload string, - impl *functions.Overload) (Interpretable, error) { + impl *functions.Overload) (InterpretableV2, error) { if impl == nil || impl.Function == nil { return nil, fmt.Errorf("no such overload: %s()", function) } @@ -324,7 +346,7 @@ func (p *planBuilder) planCallUnary(expr ast.Expr, function string, overload string, impl *functions.Overload, - args []Interpretable) (Interpretable, error) { + args []InterpretableV2) (InterpretableV2, error) { var fn functions.UnaryOp var trait int var nonStrict bool @@ -352,7 +374,7 @@ func (p *planBuilder) planCallBinary(expr ast.Expr, function string, overload string, impl *functions.Overload, - args []Interpretable) (Interpretable, error) { + args []InterpretableV2) (InterpretableV2, error) { var fn functions.BinaryOp var trait int var nonStrict bool @@ -381,7 +403,7 @@ func (p *planBuilder) planCallVarArgs(expr ast.Expr, function string, overload string, impl *functions.Overload, - args []Interpretable) (Interpretable, error) { + args []InterpretableV2) (InterpretableV2, error) { var fn functions.FunctionOp var trait int var nonStrict bool @@ -405,7 +427,7 @@ func (p *planBuilder) planCallVarArgs(expr ast.Expr, } // planCallEqual generates an equals (==) Interpretable. -func (p *planBuilder) planCallEqual(expr ast.Expr, args []Interpretable) (Interpretable, error) { +func (p *planBuilder) planCallEqual(expr ast.Expr, args []InterpretableV2) (InterpretableV2, error) { return &evalEq{ id: expr.ID(), lhs: args[0], @@ -414,7 +436,7 @@ func (p *planBuilder) planCallEqual(expr ast.Expr, args []Interpretable) (Interp } // planCallNotEqual generates a not equals (!=) Interpretable. -func (p *planBuilder) planCallNotEqual(expr ast.Expr, args []Interpretable) (Interpretable, error) { +func (p *planBuilder) planCallNotEqual(expr ast.Expr, args []InterpretableV2) (InterpretableV2, error) { return &evalNe{ id: expr.ID(), lhs: args[0], @@ -423,7 +445,7 @@ func (p *planBuilder) planCallNotEqual(expr ast.Expr, args []Interpretable) (Int } // planCallLogicalAnd generates a logical and (&&) Interpretable. -func (p *planBuilder) planCallLogicalAnd(expr ast.Expr, args []Interpretable) (Interpretable, error) { +func (p *planBuilder) planCallLogicalAnd(expr ast.Expr, args []InterpretableV2) (InterpretableV2, error) { return &evalAnd{ id: expr.ID(), terms: args, @@ -431,7 +453,7 @@ func (p *planBuilder) planCallLogicalAnd(expr ast.Expr, args []Interpretable) (I } // planCallLogicalOr generates a logical or (||) Interpretable. -func (p *planBuilder) planCallLogicalOr(expr ast.Expr, args []Interpretable) (Interpretable, error) { +func (p *planBuilder) planCallLogicalOr(expr ast.Expr, args []InterpretableV2) (InterpretableV2, error) { return &evalOr{ id: expr.ID(), terms: args, @@ -439,7 +461,7 @@ func (p *planBuilder) planCallLogicalOr(expr ast.Expr, args []Interpretable) (In } // planCallConditional generates a conditional / ternary (c ? t : f) Interpretable. -func (p *planBuilder) planCallConditional(expr ast.Expr, args []Interpretable) (Interpretable, error) { +func (p *planBuilder) planCallConditional(expr ast.Expr, args []InterpretableV2) (InterpretableV2, error) { cond := args[0] t := args[1] var tAttr Attribute @@ -467,7 +489,7 @@ func (p *planBuilder) planCallConditional(expr ast.Expr, args []Interpretable) ( // planCallIndex either extends an attribute with the argument to the index operation, or creates // a relative attribute based on the return of a function call or operation. -func (p *planBuilder) planCallIndex(expr ast.Expr, args []Interpretable, optional bool) (Interpretable, error) { +func (p *planBuilder) planCallIndex(expr ast.Expr, args []InterpretableV2, optional bool) (InterpretableV2, error) { op := args[0] ind := args[1] opType := p.typeMap[op.ID()] @@ -502,7 +524,7 @@ func (p *planBuilder) planCallIndex(expr ast.Expr, args []Interpretable, optiona } // planCreateList generates a list construction Interpretable. -func (p *planBuilder) planCreateList(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planCreateList(expr ast.Expr) (InterpretableV2, error) { list := expr.AsList() optionalIndices := list.OptionalIndices() elements := list.Elements() @@ -513,7 +535,7 @@ func (p *planBuilder) planCreateList(expr ast.Expr) (Interpretable, error) { } optionals[index] = true } - elems := make([]Interpretable, len(elements)) + elems := make([]InterpretableV2, len(elements)) for i, elem := range elements { elemVal, err := p.plan(elem) if err != nil { @@ -531,12 +553,12 @@ func (p *planBuilder) planCreateList(expr ast.Expr) (Interpretable, error) { } // planCreateStruct generates a map or object construction Interpretable. -func (p *planBuilder) planCreateMap(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planCreateMap(expr ast.Expr) (InterpretableV2, error) { m := expr.AsMap() entries := m.Entries() optionals := make([]bool, len(entries)) - keys := make([]Interpretable, len(entries)) - vals := make([]Interpretable, len(entries)) + keys := make([]InterpretableV2, len(entries)) + vals := make([]InterpretableV2, len(entries)) hasOptionals := false for i, e := range entries { entry := e.AsMapEntry() @@ -565,7 +587,7 @@ func (p *planBuilder) planCreateMap(expr ast.Expr) (Interpretable, error) { } // planCreateObj generates an object construction Interpretable. -func (p *planBuilder) planCreateStruct(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planCreateStruct(expr ast.Expr) (InterpretableV2, error) { obj := expr.AsStruct() typeName, defined := p.resolveTypeName(obj.TypeName()) if !defined { @@ -574,7 +596,7 @@ func (p *planBuilder) planCreateStruct(expr ast.Expr) (Interpretable, error) { objFields := obj.Fields() optionals := make([]bool, len(objFields)) fields := make([]string, len(objFields)) - vals := make([]Interpretable, len(objFields)) + vals := make([]InterpretableV2, len(objFields)) hasOptionals := false for i, f := range objFields { field := f.AsStructField() @@ -599,7 +621,7 @@ func (p *planBuilder) planCreateStruct(expr ast.Expr) (Interpretable, error) { } // planComprehension generates an Interpretable fold operation. -func (p *planBuilder) planComprehension(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planComprehension(expr ast.Expr) (InterpretableV2, error) { fold := expr.AsComprehension() accu, err := p.plan(fold.AccuInit()) if err != nil { @@ -639,7 +661,7 @@ func (p *planBuilder) planComprehension(expr ast.Expr) (Interpretable, error) { } // planConst generates a constant valued Interpretable. -func (p *planBuilder) planConst(expr ast.Expr) (Interpretable, error) { +func (p *planBuilder) planConst(expr ast.Expr) (InterpretableV2, error) { return NewConstValue(expr.ID(), expr.AsLiteral()), nil } @@ -726,7 +748,7 @@ func (p *planBuilder) resolveFunction(expr ast.Expr) (ast.Expr, string, string) // relativeAttr indicates that the attribute in this case acts as a qualifier and as such needs to // be observed to ensure that it's evaluation value is properly recorded for state tracking. -func (p *planBuilder) relativeAttr(id int64, eval Interpretable, opt bool) (InterpretableAttribute, error) { +func (p *planBuilder) relativeAttr(id int64, eval InterpretableV2, opt bool) (InterpretableAttribute, error) { eAttr, ok := eval.(InterpretableAttribute) if !ok { eAttr = &evalAttr{ diff --git a/vendor/github.com/google/cel-go/interpreter/runtimecost.go b/vendor/github.com/google/cel-go/interpreter/runtimecost.go index 6c44cd798..81e4ef63c 100644 --- a/vendor/github.com/google/cel-go/interpreter/runtimecost.go +++ b/vendor/github.com/google/cel-go/interpreter/runtimecost.go @@ -62,48 +62,6 @@ func CostObserver(opts ...costTrackPlanOption) PlannerOption { } } -// costTrackerConverter identifies an object which is convertible to a CostTracker instance. -type costTrackerConverter interface { - asCostTracker() *CostTracker -} - -// costTrackActivation hides state in the Activation in a manner not accessible to expressions. -type costTrackActivation struct { - vars Activation - costTracker *CostTracker -} - -// ResolveName proxies variable lookups to the backing activation. -func (cta costTrackActivation) ResolveName(name string) (any, bool) { - return cta.vars.ResolveName(name) -} - -// Parent proxies parent lookups to the backing activation. -func (cta costTrackActivation) Parent() Activation { - return cta.vars -} - -// AsPartialActivation supports conversion to a partial activation in order to detect unknown attributes. -func (cta costTrackActivation) AsPartialActivation() (PartialActivation, bool) { - return AsPartialActivation(cta.vars) -} - -// asCostTracker implements the costTrackerConverter method. -func (cta costTrackActivation) asCostTracker() *CostTracker { - return cta.costTracker -} - -// asCostTracker walks the Activation hierarchy and returns the first cost tracker found, if present. -func asCostTracker(vars Activation) (*CostTracker, bool) { - if conv, ok := vars.(costTrackerConverter); ok { - return conv.asCostTracker(), true - } - if vars.Parent() != nil { - return asCostTracker(vars.Parent()) - } - return nil, false -} - // costTrackerFactory holds a factory for producing new CostTracker instances on each Eval call. type costTrackerFactory struct { factory func() (*CostTracker, error) @@ -111,27 +69,40 @@ type costTrackerFactory struct { // InitState produces a CostTracker and bundles it into an Activation in a way which is not visible // to expression evaluation. -func (ct *costTrackerFactory) InitState(vars Activation) (Activation, error) { +func (ct *costTrackerFactory) InitState(frame *ExecutionFrame) (any, error) { + if frame.ctx != nil && frame.ctx.costs != nil { + return frame.ctx.costs, nil + } tracker, err := ct.factory() if err != nil { return nil, err } - return costTrackActivation{vars: vars, costTracker: tracker}, nil + if frame.ctx == nil { + frame.ctx = evalContextPool.Get().(*evalContext) + } + frame.ctx.costs = tracker + return tracker, nil } // GetState extracts the CostTracker from the Activation. -func (ct *costTrackerFactory) GetState(vars Activation) any { - if tracker, found := asCostTracker(vars); found { - return tracker +func (ct *costTrackerFactory) GetState(frame *ExecutionFrame) any { + if frame == nil || frame.ctx == nil { + return nil } - return nil + return frame.ctx.costs } // Observe computes the incremental cost of each step and records it into the CostTracker associated // with the evaluation. func (ct *costTrackerFactory) Observe(vars Activation, id int64, programStep any, val ref.Val) { - tracker, found := asCostTracker(vars) - if !found { + frame := AsFrame(vars) + state := ct.GetState(frame) + if state == nil { + return + } + tracker, ok := state.(*CostTracker) + if !ok { + // The state is configured with CostTrackFactory so this shouldn't happen. return } switch t := programStep.(type) { @@ -265,6 +236,19 @@ type CostTracker struct { stack refValStack } +// Clone makes a shallow copy of the tracker. +// The different clones can be used independently from +// each other. +func (c *CostTracker) Clone() (*CostTracker, error) { + tracker := &CostTracker{ + Estimator: c.Estimator, + overloadTrackers: c.overloadTrackers, + Limit: c.Limit, + presenceTestHasCost: c.presenceTestHasCost, + } + return tracker, nil +} + // ActualCost returns the runtime cost func (c *CostTracker) ActualCost() uint64 { return c.cost @@ -276,7 +260,7 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re if tracker, found := c.overloadTrackers[call.OverloadID()]; found { callCost := tracker(args, result) if callCost != nil { - cost += *callCost + cost = safeAdd(cost, *callCost) return cost } } @@ -284,7 +268,7 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re if c.Estimator != nil { callCost := c.Estimator.CallCost(call.Function(), call.OverloadID(), args, result) if callCost != nil { - cost += *callCost + cost = safeAdd(cost, *callCost) return cost } } @@ -292,12 +276,14 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re // if user has their own implementation of ActualCostEstimator, make sure to cover the mapping between overloadId and cost calculation switch call.OverloadID() { // O(n) functions - case overloads.StartsWithString, overloads.EndsWithString, overloads.StringToBytes, overloads.BytesToString, overloads.ExtQuoteString, overloads.ExtFormatString: - cost += uint64(math.Ceil(float64(actualSize(args[0])) * common.StringTraversalCostFactor)) + case overloads.StartsWithString, overloads.EndsWithString: + cost = safeAdd(cost, uint64(math.Ceil(float64(actualSize(args[1]))*common.StringTraversalCostFactor))) + case overloads.StringToBytes, overloads.BytesToString, overloads.ExtQuoteString, overloads.ExtFormatString: + cost = safeAdd(cost, uint64(math.Ceil(float64(actualSize(args[0]))*common.StringTraversalCostFactor))) case overloads.InList: // If a list is composed entirely of constant values this is O(1), but we don't account for that here. // We just assume all list containment checks are O(n). - cost += actualSize(args[1]) + cost = safeAdd(cost, actualSize(args[1])) // O(min(m, n)) functions case overloads.LessString, overloads.GreaterString, overloads.LessEqualsString, overloads.GreaterEqualsString, overloads.LessBytes, overloads.GreaterBytes, overloads.LessEqualsBytes, overloads.GreaterEqualsBytes, @@ -307,17 +293,14 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re // of 1. lhsSize := actualSize(args[0]) rhsSize := actualSize(args[1]) - minSize := lhsSize - if rhsSize < minSize { - minSize = rhsSize - } - cost += uint64(math.Ceil(float64(minSize) * common.StringTraversalCostFactor)) + minSize := min(rhsSize, lhsSize) + cost = safeAdd(cost, uint64(math.Ceil(float64(minSize)*common.StringTraversalCostFactor))) // O(m+n) functions case overloads.AddString, overloads.AddBytes: // In the worst case scenario, we would need to reallocate a new backing store and copy both operands over. - cost += uint64(math.Ceil(float64(actualSize(args[0])+actualSize(args[1])) * common.StringTraversalCostFactor)) + cost = safeAdd(cost, uint64(math.Ceil(float64(actualSize(args[0])+actualSize(args[1]))*common.StringTraversalCostFactor))) // O(nm) functions - case overloads.MatchesString: + case overloads.Matches, overloads.MatchesString: // https://swtch.com/~rsc/regexp/regexp1.html applies to RE2 implementation supported by CEL // Add one to string length for purposes of cost calculation to prevent product of string and regex to be 0 // in case where string is empty but regex is still expensive. @@ -328,11 +311,11 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re // For now, we're making a guess that each expression in a regex is typically at least 4 chars // in length. regexCost := uint64(math.Ceil(float64(actualSize(args[1])) * common.RegexStringLengthCostFactor)) - cost += strCost * regexCost + cost = safeAdd(cost, strCost*regexCost) case overloads.ContainsString: strCost := uint64(math.Ceil(float64(actualSize(args[0])) * common.StringTraversalCostFactor)) substrCost := uint64(math.Ceil(float64(actualSize(args[1])) * common.StringTraversalCostFactor)) - cost += strCost * substrCost + cost = safeAdd(cost, strCost*substrCost) default: // The following operations are assumed to have O(1) complexity. @@ -342,7 +325,7 @@ func (c *CostTracker) costCall(call InterpretableCall, args []ref.Val, result re // - Computing the size of strings, byte sequences, lists and maps. // - Logical operations and all operators on fixed width scalars (comparisons, equality) // - Any functions that don't have a declared cost either here or in provided ActualCostEstimator. - cost++ + cost = safeAdd(cost, 1) } return cost @@ -397,7 +380,7 @@ func (s *refValStack) drop(ids ...int64) { // the stack. // WARNING: It is possible for multiple expressions with the same ID to exist (due to how macros are implemented) so it's // possible that a dropped ID will remain on the stack. They should be removed when IDs on the stack are popped. -func (s *refValStack) dropArgs(args []Interpretable) ([]ref.Val, bool) { +func (s *refValStack) dropArgs(args []InterpretableV2) ([]ref.Val, bool) { result := make([]ref.Val, len(args)) argloop: for nIdx := len(args) - 1; nIdx >= 0; nIdx-- { @@ -413,3 +396,21 @@ argloop: } return result, true } + +func safeAdd(x, y uint64, rest ...uint64) uint64 { + if y > 0 && x > math.MaxUint64-y { + return math.MaxUint64 + } + next := x + y + if len(rest) == 0 { + return next + } + return safeAdd(next, rest[0], rest[1:]...) +} + +func safeMul(x, y uint64) uint64 { + if y != 0 && x > math.MaxUint64/y { + return math.MaxUint64 + } + return x * y +} diff --git a/vendor/github.com/google/cel-go/parser/helper.go b/vendor/github.com/google/cel-go/parser/helper.go index f960be20e..84bef80d5 100644 --- a/vendor/github.com/google/cel-go/parser/helper.go +++ b/vendor/github.com/google/cel-go/parser/helper.go @@ -45,6 +45,10 @@ func (p *parserHelper) getSourceInfo() *ast.SourceInfo { return p.sourceInfo } +func (p *parserHelper) expressionCount() int64 { + return p.nextID - 1 +} + func (p *parserHelper) newLiteral(ctx any, value ref.Val) ast.Expr { return p.exprFactory.NewLiteral(p.newID(ctx), value) } diff --git a/vendor/github.com/google/cel-go/parser/options.go b/vendor/github.com/google/cel-go/parser/options.go index 4eb30f83e..281021f12 100644 --- a/vendor/github.com/google/cel-go/parser/options.go +++ b/vendor/github.com/google/cel-go/parser/options.go @@ -22,6 +22,7 @@ type options struct { errorRecoveryTokenLookaheadLimit int errorRecoveryLimit int expressionSizeCodePointLimit int + maxExpressionNodeCount int macros map[string]Macro populateMacroCalls bool enableOptionalSyntax bool @@ -97,6 +98,18 @@ func ExpressionSizeCodePointLimit(expressionSizeCodePointLimit int) Option { } } +// MaxExpressionNodeCount limits the maximum number of expression nodes that may be emitted by the parser, +// including nodes created by macro expansion. +func MaxExpressionNodeCount(limit int) Option { + return func(opts *options) error { + if limit < -1 { + return fmt.Errorf("max expression node count must be greater than or equal to -1: %d", limit) + } + opts.maxExpressionNodeCount = limit + return nil + } +} + // Macros adds the given macros to the parser. func Macros(macros ...Macro) Option { return func(opts *options) error { diff --git a/vendor/github.com/google/cel-go/parser/parser.go b/vendor/github.com/google/cel-go/parser/parser.go index d1567b5ff..338233543 100644 --- a/vendor/github.com/google/cel-go/parser/parser.go +++ b/vendor/github.com/google/cel-go/parser/parser.go @@ -72,6 +72,12 @@ func NewParser(opts ...Option) (*Parser, error) { if p.expressionSizeCodePointLimit == -1 { p.expressionSizeCodePointLimit = int((^uint(0)) >> 1) } + if p.maxExpressionNodeCount == 0 { + p.maxExpressionNodeCount = 100_000 + } + if p.maxExpressionNodeCount == -1 { + p.maxExpressionNodeCount = int((^uint(0)) >> 1) + } // Bool is false by default, so populateMacroCalls will be false by default return p, nil } @@ -102,6 +108,7 @@ func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { helper: newParserHelper(source, fac), macros: p.macros, maxRecursionDepth: p.maxRecursionDepth, + maxExpressionNodeCount: p.maxExpressionNodeCount, errorReportingLimit: p.errorReportingLimit, errorRecoveryLimit: p.errorRecoveryLimit, errorRecoveryLookaheadTokenLimit: p.errorRecoveryTokenLookaheadLimit, @@ -319,6 +326,7 @@ type parser struct { recursionDepth int errorReports int maxRecursionDepth int + maxExpressionNodeCount int errorReportingLimit int errorRecoveryLimit int errorRecoveryLookaheadTokenLimit int @@ -964,11 +972,21 @@ func (p *parser) expandMacro(exprID int64, function string, target ast.Expr, arg return nil, false } } + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } eh := exprHelperPool.Get().(*exprHelper) defer exprHelperPool.Put(eh) eh.parserHelper = p.helper eh.id = exprID expr, err := macro.Expander()(eh, target, args) + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } // An error indicates that the macro was matched, but the arguments were not well-formed. if err != nil { loc := err.Location diff --git a/vendor/github.com/google/cel-go/parser/unparser.go b/vendor/github.com/google/cel-go/parser/unparser.go index ffd5b18e4..d503a450e 100644 --- a/vendor/github.com/google/cel-go/parser/unparser.go +++ b/vendor/github.com/google/cel-go/parser/unparser.go @@ -297,7 +297,7 @@ func (un *unparser) visitConstVal(val ref.Val) error { // represent the float using the minimum required digits d := strconv.FormatFloat(float64(val), 'g', -1, 64) un.str.WriteString(d) - if !strings.Contains(d, ".") { + if !strings.ContainsAny(d, ".eE") { un.str.WriteString(".0") } case types.Int: diff --git a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go b/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go deleted file mode 100644 index 69021bcee..000000000 --- a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package estargz adapts the containerd estargz package to our abstractions. -package estargz - -import ( - "bytes" - "io" - - "github.com/containerd/stargz-snapshotter/estargz" - v1 "github.com/google/go-containerregistry/pkg/v1" -) - -// Assert that what we're returning is an io.ReadCloser -var _ io.ReadCloser = (*estargz.Blob)(nil) - -// ReadCloser reads uncompressed tarball input from the io.ReadCloser and -// returns: -// - An io.ReadCloser from which compressed data may be read, and -// - A v1.Hash with the hash of the estargz table of contents, or -// - An error if the estargz processing encountered a problem. -// -// Refer to estargz for the options: -// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option -func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) { - defer r.Close() - - // TODO(#876): Avoid buffering into memory. - bs, err := io.ReadAll(r) - if err != nil { - return nil, v1.Hash{}, err - } - br := bytes.NewReader(bs) - - rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...) - if err != nil { - return nil, v1.Hash{}, err - } - - h, err := v1.NewHash(rc.TOCDigest().String()) - return rc, h, err -} diff --git a/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go b/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go index 018c0f8c0..98100c024 100644 --- a/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go +++ b/vendor/github.com/google/go-containerregistry/internal/gzip/zip.go @@ -51,33 +51,31 @@ func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser { // Returns err so we can pw.CloseWithError(err) go func() error { - // TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect. - // Context: https://golang.org/issue/24283 + // Always close the source reader when the goroutine exits, + // regardless of which error path is taken. This prevents + // resource leaks (e.g. pullLimiter token slots held by + // limitedReadCloser wrappers around r). + defer r.Close() + gw, err := gzip.NewWriterLevel(bw, level) if err != nil { return pw.CloseWithError(err) } + defer gw.Close() + defer pw.Close() if _, err := io.Copy(gw, r); err != nil { - defer r.Close() - defer gw.Close() return pw.CloseWithError(err) } - // Close gzip writer to Flush it and write gzip trailers. if err := gw.Close(); err != nil { return pw.CloseWithError(err) } - // Flush bufio writer to ensure we write out everything. if err := bw.Flush(); err != nil { return pw.CloseWithError(err) } - // We don't really care if these fail. - defer pw.Close() - defer r.Close() - return nil }() diff --git a/vendor/github.com/google/go-containerregistry/internal/ipaddr/ipaddr.go b/vendor/github.com/google/go-containerregistry/internal/ipaddr/ipaddr.go new file mode 100644 index 000000000..a7f5b8a4b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/internal/ipaddr/ipaddr.go @@ -0,0 +1,82 @@ +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipaddr + +import ( + "net/netip" + "strconv" + "strings" +) + +// IsPrivateOrLinkLocal reports whether host denotes a loopback, private, +// link-local, or unspecified address. It accepts any IP-literal form the Go +// dialer accepts — canonical dotted-quad IPv4 and IPv6, zone-qualified and +// IPv4-mapped IPv6, and legacy inet_aton encodings (32-bit decimal +// "2130706433", hexadecimal "0x7f000001", partial dotted-quad "127.1", +// zero-padded octets) — so a guard based on it cannot be bypassed by +// spelling an internal address in a non-canonical way. DNS names are not IP +// literals and return false. +func IsPrivateOrLinkLocal(host string) bool { + addr, ok := Parse(host) + if !ok { + return false + } + return addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || + addr.IsPrivate() || addr.IsUnspecified() +} + +// Parse parses an IP literal in any form the Go dialer accepts. +func Parse(host string) (netip.Addr, bool) { + // The dialer ignores IPv6 zone identifiers, so guards must too. + host, _, _ = strings.Cut(host, "%") + if addr, err := netip.ParseAddr(host); err == nil { + // Treat IPv4-mapped IPv6 as the IPv4 address the dialer connects to. + return addr.WithZone("").Unmap(), true + } + return parseLegacyIPv4(host) +} + +// parseLegacyIPv4 implements the inet_aton forms the Go resolver accepts for +// IPv4: one to four dot-separated parts where the final part may fill the +// remaining bytes ("127.1" == 127.0.0.1) and each part may be decimal, +// hexadecimal (0x prefix), or octal (leading 0). +func parseLegacyIPv4(host string) (netip.Addr, bool) { + parts := strings.Split(host, ".") + if len(parts) < 1 || len(parts) > 4 { + return netip.Addr{}, false + } + var b [4]byte + for i, part := range parts[:len(parts)-1] { + // bitsize 8 enforces the single-octet range before any conversion. + v, err := strconv.ParseUint(part, 0, 8) + if err != nil { + return netip.Addr{}, false + } + b[i] = byte(v) + } + // The final part may fill as many bytes as remain, e.g. "127.1" -> 127.0.0.1; + // with four parts it must still be a single octet. The bitsize enforces the + // range before the conversion below. + lastBits := [...]int{32, 24, 16, 8}[len(parts)-1] + last, err := strconv.ParseUint(parts[len(parts)-1], 0, lastBits) + if err != nil { + return netip.Addr{}, false + } + v := uint32(last) + for i := len(parts) - 1; i < 4; i++ { + b[i] = byte(v >> (8 * (3 - i))) + } + return netip.AddrFrom4(b), true +} diff --git a/vendor/github.com/google/go-containerregistry/internal/limit/limit.go b/vendor/github.com/google/go-containerregistry/internal/limit/limit.go new file mode 100644 index 000000000..29d2b08ac --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/internal/limit/limit.go @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package limit provides bounded reads from io.Readers. +package limit + +import ( + "fmt" + "io" +) + +// ReadAll reads from r until EOF and returns the data. If r produces more +// than max bytes, it returns an error instead of a silently truncated +// slice. Use this in preference to io.ReadAll(io.LimitReader(r, max)) +// when callers must distinguish a complete read from a truncated one. +func ReadAll(r io.Reader, max int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, max+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > max { + return nil, fmt.Errorf("body exceeds %d byte limit", max) + } + return b, nil +} diff --git a/vendor/github.com/google/go-containerregistry/internal/verify/verify.go b/vendor/github.com/google/go-containerregistry/internal/verify/verify.go index 463f7e4b3..ff48ba871 100644 --- a/vendor/github.com/google/go-containerregistry/internal/verify/verify.go +++ b/vendor/github.com/google/go-containerregistry/internal/verify/verify.go @@ -17,7 +17,6 @@ package verify import ( - "bytes" "encoding/hex" "errors" "fmt" @@ -107,10 +106,16 @@ func Descriptor(d v1.Descriptor) error { return errors.New("error verifying descriptor; Data == nil") } - h, sz, err := v1.SHA256(bytes.NewReader(d.Data)) + hasher, err := v1.Hasher(d.Digest.Algorithm) if err != nil { return err } + hasher.Write(d.Data) + h := v1.Hash{ + Algorithm: d.Digest.Algorithm, + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + } + sz := int64(len(d.Data)) if h != d.Digest { return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/authn.go b/vendor/github.com/google/go-containerregistry/pkg/authn/authn.go index 1555efae0..c962235c7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/authn.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/authn.go @@ -89,7 +89,9 @@ func (a *AuthConfig) UnmarshalJSON(data []byte) error { // MarshalJSON implements json.Marshaler func (a AuthConfig) MarshalJSON() ([]byte, error) { shadow := (authConfig)(a) - shadow.Auth = encodeDockerConfigFieldAuth(shadow.Username, shadow.Password) + if shadow.Username != "" || shadow.Password != "" { + shadow.Auth = encodeDockerConfigFieldAuth(shadow.Username, shadow.Password) + } return json.Marshal(shadow) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index 6f08460ba..a9b105e17 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -25,7 +25,6 @@ import ( "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/config/types" "github.com/google/go-containerregistry/pkg/name" - "github.com/mitchellh/go-homedir" ) // Resource represents a registry or repository that can be authenticated against. @@ -95,7 +94,7 @@ func (dk *defaultKeychain) ResolveContext(_ context.Context, target Resource) (A // First, check $HOME/.docker/config.json foundDockerConfig := false - home, err := homedir.Dir() + home, err := os.UserHomeDir() if err == nil { foundDockerConfig = fileExists(filepath.Join(home, ".docker/config.json")) } @@ -103,11 +102,19 @@ func (dk *defaultKeychain) ResolveContext(_ context.Context, target Resource) (A if !foundDockerConfig && os.Getenv("DOCKER_CONFIG") != "" { foundDockerConfig = fileExists(filepath.Join(os.Getenv("DOCKER_CONFIG"), "config.json")) } + configDir := os.Getenv("XDG_CONFIG_HOME") + if configDir == "" && home != "" { + configDir = filepath.Join(home, ".config") + } + podmanAuth := filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "containers/auth.json") + if (os.Getenv("XDG_RUNTIME_DIR") == "" || !fileExists(podmanAuth)) && configDir != "" { + podmanAuth = filepath.Join(configDir, "containers/auth.json") + } // If either of those locations are found, load it using Docker's // config.Load, which may fail if the config can't be parsed. // // If neither was found, look for Podman's auth at - // $REGISTRY_AUTH_FILE or $XDG_RUNTIME_DIR/containers/auth.json + // $REGISTRY_AUTH_FILE or containers/auth.json under XDG runtime/config dirs // and attempt to load it as a Docker config. // // If neither are found, fallback to Anonymous. @@ -127,7 +134,7 @@ func (dk *defaultKeychain) ResolveContext(_ context.Context, target Resource) (A if err != nil { return nil, err } - } else if path := filepath.Clean(filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "containers/auth.json")); fileExists(path) { + } else if path := filepath.Clean(podmanAuth); fileExists(path) { f, err := os.Open(path) if err != nil { return nil, err diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go index 5b8eb4ff4..9d6650ee6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go @@ -17,6 +17,8 @@ package name import ( // nolint: depguard _ "crypto/sha256" // Recommended by go-digest. + // nolint: depguard + _ "crypto/sha512" // Needed for sha512 digests. "encoding" "encoding/json" "strings" @@ -107,13 +109,8 @@ func NewDigest(name string, opts ...Option) (Digest, error) { } base := parts[0] dig := parts[1] - prefix := digest.Canonical.String() + ":" - if !strings.HasPrefix(dig, prefix) { - return Digest{}, newErrBadName("unsupported digest algorithm: %s", dig) - } - hex := strings.TrimPrefix(dig, prefix) - if err := digest.Canonical.Validate(hex); err != nil { - return Digest{}, err + if err := digest.Digest(dig).Validate(); err != nil { + return Digest{}, newErrBadName("%s: %s", err, dig) } tag, err := NewTag(base, opts...) diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go index b294794dc..b67ad0ffc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go @@ -17,9 +17,11 @@ // What's in a name? For image references, not nearly enough! // // Image references look a lot like URLs, but they differ in that they don't -// contain the scheme (http or https), they can end with a :tag or a @digest -// (the latter being validated), and they perform defaulting for missing -// components. +// contain the scheme (http or https), they can end with a :tag, an @digest +// (the latter being validated), or both. When a reference includes both a tag +// and a digest, it is treated as a digest reference: the digest is the +// identifier used for verification, while the tag is retained only in the +// original string form. // // Since image references don't contain the scheme, we do our best to infer // if we use http or https from the given hostname. We allow http fallback for diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go index 0a0486772..1ce999a2e 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go @@ -16,6 +16,7 @@ package name import ( "fmt" + "strings" ) // Reference defines the interface that consumers use when they can @@ -37,7 +38,13 @@ type Reference interface { } // ParseReference parses the string as a reference, either by tag or digest. +// References that include both a tag and digest parse as Digest references. func ParseReference(s string, opts ...Option) (Reference, error) { + // Image references never contain a URL scheme, so tell the user to + // strip it instead of returning a confusing parse error. + if strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") { + return nil, newErrBadName("image reference must not contain a URL scheme (http:// or https://): %s; to connect to a registry over plain HTTP, use name.Insecure", s) + } if t, err := NewTag(s, opts...); err == nil { return t, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index 5e6b6e62a..478021486 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -24,14 +24,14 @@ import ( "strings" ) -// Detect more complex forms of local references. -var reLocal = regexp.MustCompile(`.*\.local(?:host)?(?::\d{1,5})?$`) +// Detect more complex forms of localhost references. +var reLocal = regexp.MustCompile(`.*\.localhost(?::\d{1,5})?$`) // Detect the loopback IP (127.0.0.1) -var reLoopback = regexp.MustCompile(regexp.QuoteMeta("127.0.0.1")) +var reLoopback = regexp.MustCompile(`^127\.0\.0\.1(?::\d{1,5})?$`) // Detect the loopback IPV6 (::1) -var reipv6Loopback = regexp.MustCompile(regexp.QuoteMeta("::1")) +var reipv6Loopback = regexp.MustCompile(`^(::1|\[::1\](?::\d{1,5})?)$`) // Registry stores a docker registry name in a structured form. type Registry struct { diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go index 290797575..ad7b32911 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go @@ -72,7 +72,7 @@ func (r Repository) Scope(action string) string { } func checkRepository(repository string) error { - return checkElement("repository", repository, repositoryChars, 2, 255) + return checkElement("repository", repository, repositoryChars, 1, 255) } // NewRepository returns a new Repository representing the given name, according to the given strictness. @@ -85,11 +85,12 @@ func NewRepository(name string, opts ...Option) (Repository, error) { var registry string repo := name parts := strings.SplitN(name, regRepoDelimiter, 2) - if len(parts) == 2 && (strings.ContainsRune(parts[0], '.') || strings.ContainsRune(parts[0], ':')) { + maybeRegistry := parts[0] + if len(parts) == 2 && (maybeRegistry == "localhost" || strings.ContainsAny(maybeRegistry, ".:")) { // The first part of the repository is treated as the registry domain - // iff it contains a '.' or ':' character, otherwise it is all repository - // and the domain defaults to Docker Hub. - registry = parts[0] + // if it is localhost or contains a '.' or ':' character, otherwise it + // is all repository and the domain defaults to Docker Hub. + registry = maybeRegistry repo = parts[1] } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go b/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go index bbb600ed7..4114d5ab2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/hash.go @@ -16,6 +16,10 @@ package v1 import ( "crypto" + // nolint: depguard + _ "crypto/sha256" // Registered for Hasher. + // nolint: depguard + _ "crypto/sha512" // Registered for Hasher. "encoding" "encoding/hex" "encoding/json" @@ -78,6 +82,8 @@ func Hasher(name string) (hash.Hash, error) { switch name { case "sha256": return crypto.SHA256.New(), nil + case "sha512": + return crypto.SHA512.New(), nil default: return nil, fmt.Errorf("unsupported hash: %q", name) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go index 22d483f3b..783eecc69 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go @@ -29,6 +29,7 @@ type Manifest struct { Layers []Descriptor `json:"layers"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // IndexManifest represents an OCI image index in a structured way. @@ -38,6 +39,7 @@ type IndexManifest struct { Manifests []Descriptor `json:"manifests"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // Descriptor holds a reference from the manifest to one of its constituent elements. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go index 3ea27fe47..0c58e0c1b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go @@ -52,6 +52,16 @@ func (i *image) MediaType() (types.MediaType, error) { return i.base.MediaType() } +// isImageConfig reports whether the media type is a Docker or OCI image config. +func isImageConfig(mt types.MediaType) bool { + switch mt { + case types.DockerConfigJSON, types.OCIConfigJSON: + return true + default: + return false + } +} + func (i *image) compute() error { i.Lock() defer i.Unlock() @@ -60,6 +70,24 @@ func (i *image) compute() error { if i.computed { return nil } + + m, err := i.base.Manifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + + diffIDMap := make(map[v1.Hash]v1.Layer) + digestMap := make(map[v1.Hash]v1.Layer) + + // Determine effective config media type (user override takes precedence). + cfgMediaType := manifest.Config.MediaType + if i.configMediaType != nil { + cfgMediaType = *i.configMediaType + } + + imageConfig := isImageConfig(cfgMediaType) + var configFile *v1.ConfigFile if i.configFile != nil { configFile = i.configFile @@ -70,33 +98,32 @@ func (i *image) compute() error { } configFile = cf.DeepCopy() } - diffIDs := configFile.RootFS.DiffIDs - history := configFile.History - diffIDMap := make(map[v1.Hash]v1.Layer) - digestMap := make(map[v1.Hash]v1.Layer) + // For image configs, update RootFS.DiffIDs and History from added layers. + // For artifacts, skip this: the config has no rootfs or history fields. + if imageConfig { + diffIDs := configFile.RootFS.DiffIDs + history := configFile.History - for _, add := range i.adds { - history = append(history, add.History) - if add.Layer != nil { - diffID, err := add.Layer.DiffID() - if err != nil { - return err + for _, add := range i.adds { + history = append(history, add.History) + if add.Layer != nil { + diffID, err := add.Layer.DiffID() + if err != nil { + return err + } + diffIDs = append(diffIDs, diffID) + diffIDMap[diffID] = add.Layer } - diffIDs = append(diffIDs, diffID) - diffIDMap[diffID] = add.Layer } - } - m, err := i.base.Manifest() - if err != nil { - return err + configFile.RootFS.DiffIDs = diffIDs + configFile.History = history } - manifest := m.DeepCopy() + manifestLayers := manifest.Layers for _, add := range i.adds { if add.Layer == nil { - // Empty layers include only history in manifest. continue } @@ -105,14 +132,12 @@ func (i *image) compute() error { return err } - // Fields in the addendum override the original descriptor. if len(add.Annotations) != 0 { desc.Annotations = add.Annotations } if len(add.URLs) != 0 { desc.URLs = add.URLs } - if add.MediaType != "" { desc.MediaType = add.MediaType } @@ -120,42 +145,38 @@ func (i *image) compute() error { manifestLayers = append(manifestLayers, *desc) digestMap[desc.Digest] = add.Layer } - - configFile.RootFS.DiffIDs = diffIDs - configFile.History = history - manifest.Layers = manifestLayers - rcfg, err := json.Marshal(configFile) - if err != nil { - return err - } - d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) - if err != nil { - return err - } - manifest.Config.Digest = d - manifest.Config.Size = sz + // For image configs, re-marshal the config and update the manifest digest. + // For artifacts, preserve the original config blob as-is to avoid + // corrupting the digest via re-marshaling. + if imageConfig { + rcfg, err := json.Marshal(configFile) + if err != nil { + return err + } + d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) + if err != nil { + return err + } + manifest.Config.Digest = d + manifest.Config.Size = sz - // If Data was set in the base image, we need to update it in the mutated image. - if m.Config.Data != nil { - manifest.Config.Data = rcfg + if m.Config.Data != nil { + manifest.Config.Data = rcfg + } } - // If the user wants to mutate the media type of the config if i.configMediaType != nil { manifest.Config.MediaType = *i.configMediaType } - if i.mediaType != nil { manifest.MediaType = *i.mediaType } - if i.annotations != nil { if manifest.Annotations == nil { manifest.Annotations = map[string]string{} } - for k, v := range i.annotations { manifest.Annotations[k] = v } @@ -173,29 +194,34 @@ func (i *image) compute() error { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. func (i *image) Layers() ([]v1.Layer, error) { - if err := i.compute(); errors.Is(err, stream.ErrNotComputed) { - // Image contains a streamable layer which has not yet been - // consumed. Just return the layers we have in case the caller - // is going to consume the layers. + if err := i.compute(); errors.Is(err, stream.ErrNotComputed) || (i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType)) { + // Stream not yet consumed, or non-image OCI artifact (RootFS.DiffIDs + // is empty so partial.DiffIDs returns nothing). Fall back to the base + // layers plus any added layers. layers, err := i.base.Layers() if err != nil { return nil, err } for _, add := range i.adds { - layers = append(layers, add.Layer) + if add.Layer != nil { + layers = append(layers, add.Layer) + } } return layers, nil } else if err != nil { return nil, err } - diffIDs, err := partial.DiffIDs(i) - if err != nil { - return nil, err - } - ls := make([]v1.Layer, 0, len(diffIDs)) - for _, h := range diffIDs { - l, err := i.LayerByDiffID(h) + // Walk manifest layer descriptors by digest rather than rootfs diff + // IDs. Two layers can legitimately share a diff ID — same uncompressed + // content, different compression — and produce distinct digests. The + // manifest preserves the per-occurrence digest; LayerByDiffID does not, + // which previously caused duplicate-diff-ID layers to collapse to a + // single entry in the returned slice and break blob upload for + // downstream pushers (see #2034). + ls := make([]v1.Layer, 0, len(i.manifest.Layers)) + for _, desc := range i.manifest.Layers { + l, err := i.LayerByDigest(desc.Digest) if err != nil { return nil, err } @@ -220,11 +246,18 @@ func (i *image) ConfigFile() (*v1.ConfigFile, error) { return i.configFile.DeepCopy(), nil } -// RawConfigFile returns the serialized bytes of ConfigFile() +// RawConfigFile returns the serialized bytes of ConfigFile(). +// For non-image OCI artifacts, returns the original raw config to preserve +// the config blob digest. func (i *image) RawConfigFile() ([]byte, error) { if err := i.compute(); err != nil { return nil, err } + // If the manifest config is not a standard image config, return the + // original raw bytes to avoid corrupting the digest via re-marshaling. + if i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType) { + return i.base.RawConfigFile() + } return json.Marshal(i.configFile) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go index a6fdaceed..7b4ed0884 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go @@ -56,6 +56,9 @@ func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) { if ia.Data != nil { desc.Data = ia.Data } + if ia.ArtifactType != "" { + desc.ArtifactType = ia.ArtifactType + } return desc, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 2e35f2250..00ad8c0d4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -22,11 +22,11 @@ import ( "fmt" "io" "maps" - "path/filepath" + "path" "strings" + "sync" "time" - "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/match" @@ -37,6 +37,11 @@ import ( const whiteoutPrefix = ".wh." +// opaqueWhiteout marks a directory as opaque: all entries from lower layers under +// its parent directory are hidden. It shares the whiteoutPrefix, so it must be +// matched exactly and handled before the generic per-file whiteout logic. +const opaqueWhiteout = ".wh..wh..opq" + // Addendum contains layers and history to be appended // to a base image type Addendum struct { @@ -267,6 +272,9 @@ func extract(img v1.Image, w io.Writer) error { defer tarWriter.Close() fileMap := map[string]bool{} + // opaqueDirs holds directories opaqued by an upper layer; entries under them + // from lower (later-iterated) layers are hidden. + opaqueDirs := map[string]bool{} layers, err := img.Layers() if err != nil { @@ -277,77 +285,145 @@ func extract(img v1.Image, w io.Writer) error { // whiteout layers more efficient, since we can just keep track of the removed // files as we see .wh. layers and ignore those in previous layers. for i := len(layers) - 1; i >= 0; i-- { - layer := layers[i] - layerReader, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("reading layer contents: %w", err) + if err := extractLayer(tarWriter, fileMap, opaqueDirs, layers[i]); err != nil { + return err } - defer layerReader.Close() - tarReader := tar.NewReader(layerReader) - for { - header, err := tarReader.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("reading tar: %w", err) - } + } + return nil +} - // Some tools prepend everything with "./", so if we don't Clean the - // name, we may have duplicate entries, which angers tar-split. - header.Name = filepath.Clean(header.Name) +func extractLayer(tarWriter *tar.Writer, fileMap, opaqueDirs map[string]bool, layer v1.Layer) error { + // Opaque markers in this layer hide only lower layers, so stage them and + // promote to opaqueDirs after the whole layer is processed. + layerOpaque := map[string]bool{} - // force PAX format to remove Name/Linkname length limit of 100 characters - // required by USTAR and to not depend on internal tar package guess which - // prefers USTAR over PAX - header.Format = tar.FormatPAX + layerReader, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("reading layer contents: %w", err) + } + defer layerReader.Close() - basename := filepath.Base(header.Name) - dirname := filepath.Dir(header.Name) - tombstone := strings.HasPrefix(basename, whiteoutPrefix) - if tombstone { - basename = basename[len(whiteoutPrefix):] - } + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading tar: %w", err) + } - // check if we have seen value before - // if we're checking a directory, don't filepath.Join names - var name string - if header.Typeflag == tar.TypeDir { - name = header.Name - } else { - name = filepath.Join(dirname, basename) - } + // Some tools prepend everything with "./", so if we don't Clean the + // name, we may have duplicate entries, which angers tar-split. + header.Name = path.Clean(header.Name) + if unsafeArchivePath(header.Name) { + return fmt.Errorf("unsafe tar path %q", header.Name) + } - if _, ok := fileMap[name]; ok && !tombstone { + // Reject relative symlinks and hardlinks whose targets escape the + // image rootfs. Relative targets are resolved against the symlink's + // own directory: if the clean result starts with ".." the link would + // leave the rootfs. Relative symlinks that stay within the rootfs + // (common for glibc, C toolchains, etc.) are preserved unchanged. + // Absolute targets are left as-is; see #2238 for ongoing discussion + // on whether they should be pruned. + if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink { + if unsafeRelativeLink(header.Name, header.Linkname) { continue } + } - // check for a whited out parent directory - if inWhiteoutDir(fileMap, name) { - continue - } + // force PAX format to remove Name/Linkname length limit of 100 characters + // required by USTAR and to not depend on internal tar package guess which + // prefers USTAR over PAX + header.Format = tar.FormatPAX - // mark file as handled. non-directory implicitly tombstones - // any entries with a matching (or child) name - fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) - if !tombstone { - if err := tarWriter.WriteHeader(header); err != nil { + basename := path.Base(header.Name) + dirname := path.Dir(header.Name) + + // An opaque marker hides all lower-layer entries under dirname. It shares + // the whiteout prefix, so handle it before the generic per-file logic. + if basename == opaqueWhiteout { + layerOpaque[dirname] = true + continue + } + + tombstone := strings.HasPrefix(basename, whiteoutPrefix) + if tombstone { + basename = basename[len(whiteoutPrefix):] + } + + // check if we have seen value before + // if we're checking a directory, don't filepath.Join names + var name string + if header.Typeflag == tar.TypeDir { + name = header.Name + } else { + name = path.Join(dirname, basename) + } + + if _, ok := fileMap[name]; ok && !tombstone { + continue + } + + // check for a whited out parent directory + if inWhiteoutDir(fileMap, name) { + continue + } + + // check for a parent directory opaqued by an upper layer + if inOpaqueDir(opaqueDirs, name) { + continue + } + + // mark file as handled. non-directory implicitly tombstones + // any entries with a matching (or child) name + fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) + if !tombstone { + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if header.Size > 0 { + if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { return err } - if header.Size > 0 { - if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { - return err - } - } } } } + + // Opaque dirs found in this layer now hide entries in lower layers. + for d := range layerOpaque { + opaqueDirs[d] = true + } + + // Drain any bytes the tar.Reader did not consume (trailing data after the + // end-of-archive marker) so the underlying verifying reader reaches io.EOF + // and the layer's digest is verified. Without this, a layer whose contents + // do not match the manifest's layer digest is extracted without error. + // pkg/v1/validate/layer.go performs the same drain. + if _, err := io.Copy(io.Discard, layerReader); err != nil { + return fmt.Errorf("verifying layer: %w", err) + } return nil } +func inOpaqueDir(opaqueDirs map[string]bool, file string) bool { + for file != "" { + dirname := path.Dir(file) + if file == dirname { + break + } + if opaqueDirs[dirname] { + return true + } + file = dirname + } + return false +} + func inWhiteoutDir(fileMap map[string]bool, file string) bool { for file != "" { - dirname := filepath.Dir(file) + dirname := path.Dir(file) if file == dirname { break } @@ -359,8 +435,42 @@ func inWhiteoutDir(fileMap map[string]bool, file string) bool { return false } +func unsafeArchivePath(name string) bool { + clean := cleanArchivePath(name) + if clean == ".." || strings.HasPrefix(clean, "../") { + return true + } + if strings.HasPrefix(name, "\\") { + return true + } + return hasWindowsDrivePrefix(clean) +} + +func unsafeRelativeLink(name, linkname string) bool { + clean := cleanArchivePath(linkname) + if path.IsAbs(clean) { + return false + } + if hasWindowsDrivePrefix(clean) { + return true + } + resolved := path.Clean(path.Join(path.Dir(name), clean)) //nolint:gosec // G305: path is only used for validation, not file I/O + return strings.HasPrefix(resolved, "..") +} + +func cleanArchivePath(name string) string { + return path.Clean(strings.ReplaceAll(name, "\\", "/")) +} + +func hasWindowsDrivePrefix(name string) bool { + return len(name) >= 2 && + (('A' <= name[0] && name[0] <= 'Z') || ('a' <= name[0] && name[0] <= 'z')) && + name[1] == ':' +} + // Time sets all timestamps in an image to the given timestamp. -func Time(img v1.Image, t time.Time) (v1.Image, error) { +// Layers are rewritten as dockerv2+gzip unless opts say otherwise. +func Time(img v1.Image, t time.Time, opts ...tarball.LayerOption) (v1.Image, error) { newImage := empty.Image layers, err := img.Layers() @@ -376,10 +486,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { addendums := make([]Addendum, max(len(ocf.History), len(layers))) var historyIdx, addendumIdx int for layerIdx := 0; layerIdx < len(layers); addendumIdx, layerIdx = addendumIdx+1, layerIdx+1 { - newLayer, err := layerTime(layers[layerIdx], t) - if err != nil { - return nil, fmt.Errorf("setting layer times: %w", err) - } + newLayer := layerTime(layers[layerIdx], t, opts...) // try to search for the history entry that corresponds to this layer for ; historyIdx < len(ocf.History); historyIdx++ { @@ -438,7 +545,28 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { return ConfigFile(newImage, cfg) } -func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { +func layerTime(layer v1.Layer, t time.Time, opts ...tarball.LayerOption) v1.Layer { + return &timeLayer{inner: layer, t: t, opts: opts} +} + +type timeLayer struct { + inner v1.Layer + t time.Time + opts []tarball.LayerOption + + once sync.Once + material v1.Layer + err error +} + +func (l *timeLayer) materialize() error { + l.once.Do(func() { + l.material, l.err = materializeLayerTime(l.inner, l.t, l.opts...) + }) + return l.err +} + +func materializeLayerTime(layer v1.Layer, t time.Time, opts ...tarball.LayerOption) (v1.Layer, error) { layerReader, err := layer.Uncompressed() if err != nil { return nil, fmt.Errorf("getting layer: %w", err) @@ -471,36 +599,82 @@ func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { } if header.Typeflag == tar.TypeReg { - // TODO(#1168): This should be lazy, and not buffer the entire layer contents. if _, err = io.CopyN(tarWriter, tarReader, header.Size); err != nil { return nil, fmt.Errorf("writing layer file: %w", err) } } } + // Drain trailing bytes so the underlying verifying reader reaches io.EOF + // and the layer digest is verified (see extractLayer). + if _, err := io.Copy(io.Discard, layerReader); err != nil { + return nil, fmt.Errorf("verifying layer: %w", err) + } + if err := tarWriter.Close(); err != nil { return nil, err } b := w.Bytes() - // gzip the contents, then create the layer opener := func() (io.ReadCloser, error) { - return gzip.ReadCloser(io.NopCloser(bytes.NewReader(b))), nil + return io.NopCloser(bytes.NewReader(b)), nil } - layer, err = tarball.LayerFromOpener(opener) + newLayer, err := tarball.LayerFromOpener(opener, opts...) if err != nil { return nil, fmt.Errorf("creating layer: %w", err) } - return layer, nil + return newLayer, nil +} + +func (l *timeLayer) Compressed() (io.ReadCloser, error) { + if err := l.materialize(); err != nil { + return nil, err + } + return l.material.Compressed() +} + +func (l *timeLayer) Uncompressed() (io.ReadCloser, error) { + if err := l.materialize(); err != nil { + return nil, err + } + return l.material.Uncompressed() +} + +func (l *timeLayer) Size() (int64, error) { + if err := l.materialize(); err != nil { + return 0, err + } + return l.material.Size() +} + +func (l *timeLayer) DiffID() (v1.Hash, error) { + if err := l.materialize(); err != nil { + return v1.Hash{}, err + } + return l.material.DiffID() +} + +func (l *timeLayer) Digest() (v1.Hash, error) { + if err := l.materialize(); err != nil { + return v1.Hash{}, err + } + return l.material.Digest() +} + +func (l *timeLayer) MediaType() (types.MediaType, error) { + if err := l.materialize(); err != nil { + return "", err + } + return l.material.MediaType() } // Canonical is a helper function to combine Time and configFile // to remove any randomness during a docker build. -func Canonical(img v1.Image) (v1.Image, error) { +func Canonical(img v1.Image, opts ...tarball.LayerOption) (v1.Image, error) { // Set all timestamps to 0 created := time.Time{} - img, err := Time(img, created) + img, err := Time(img, created, opts...) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go index c606e0b76..a7ac8ff53 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go @@ -97,13 +97,21 @@ func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) { return nil, fmt.Errorf("could not get new base layers for new base: %w", err) } // Add new base layers. - rebasedImage, err = Append(rebasedImage, createAddendums(0, 0, newConfig.History, newBaseLayers)...) + newBaseAdds, err := createAddendums(0, 0, newConfig.History, newBaseLayers) + if err != nil { + return nil, fmt.Errorf("failed to process new base image: %w", err) + } + rebasedImage, err = Append(rebasedImage, newBaseAdds...) if err != nil { return nil, fmt.Errorf("failed to append new base image: %w", err) } // Add original layers above the old base. - rebasedImage, err = Append(rebasedImage, createAddendums(len(oldConfig.History), len(oldBaseLayers)+1, origConfig.History, origLayers)...) + origAdds, err := createAddendums(len(oldConfig.History), len(oldBaseLayers)+1, origConfig.History, origLayers) + if err != nil { + return nil, fmt.Errorf("failed to process original image: %w", err) + } + rebasedImage, err = Append(rebasedImage, origAdds...) if err != nil { return nil, fmt.Errorf("failed to append original image: %w", err) } @@ -113,7 +121,7 @@ func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) { // createAddendums makes a list of addendums from a history and layers starting from a specific history and layer // indexes. -func createAddendums(startHistory, startLayer int, history []v1.History, layers []v1.Layer) []Addendum { +func createAddendums(startHistory, startLayer int, history []v1.History, layers []v1.Layer) ([]Addendum, error) { var adds []Addendum // History should be a superset of layers; empty layers (e.g. ENV statements) only exist in history. // They cannot be iterated identically but must be walked independently, only advancing the iterator for layers @@ -123,6 +131,9 @@ func createAddendums(startHistory, startLayer int, history []v1.History, layers var layer v1.Layer emptyLayer := history[historyIndex].EmptyLayer if !emptyLayer { + if layerIndex >= len(layers) { + return nil, fmt.Errorf("malformed image: config history references at least %d non-empty layer(s), but only %d layer(s) are present", layerIndex+1, len(layers)) + } layer = layers[layerIndex] layerIndex++ } @@ -136,9 +147,9 @@ func createAddendums(startHistory, startLayer int, history []v1.History, layers // In the event history was malformed or non-existent, append the remaining layers. for i := layerIndex; i < len(layers); i++ { if i >= startLayer { - adds = append(adds, Addendum{Layer: layers[layerIndex]}) + adds = append(adds, Addendum{Layer: layers[i]}) } } - return adds + return adds, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index c8b22b3f9..e83b4a882 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -337,8 +337,12 @@ func Descriptor(d Describable) (*v1.Descriptor, error) { mf, _ := Manifest(wrm) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - desc.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) + } } } } @@ -429,7 +433,10 @@ func ArtifactType(w WithManifest) (string, error) { mf, _ := w.Manifest() // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { + if mf != nil { + if mf.ArtifactType != "" { + return mf.ArtifactType, nil + } return string(mf.Config.MediaType), nil } return "", nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/catalog.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/catalog.go index a0281b9fd..7b9cc0e24 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/catalog.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/catalog.go @@ -122,7 +122,7 @@ func (f *fetcher) catalogPage(ctx context.Context, reg name.Registry, next strin return nil, err } - uri, err := getNextPageURL(resp) + uri, err := getNextPageURLForRegistry(resp, reg) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index b4395c239..06e909908 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -32,9 +32,9 @@ import ( // // TODO(#412): Remove the need for this method. func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { - auth, err := kc.Resolve(ref.Context().Registry) + auth, err := authn.Resolve(context.TODO(), kc, ref.Context()) if err != nil { - return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context().Registry, err) + return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context(), err) } scopes := []string{ref.Scope(transport.PushScope)} @@ -50,7 +50,7 @@ func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTrip // to avoid a roundtrip for spec-compliant registries. w := writer{ repo: ref.Context(), - client: &http.Client{Transport: tr}, + client: &http.Client{Transport: tr, CheckRedirect: checkRedirectSSRF}, } loc, _, err := w.initiateUpload(context.Background(), "", "", "") if loc != "" { @@ -68,5 +68,5 @@ func (w *writer) cancelUpload(loc string) { if err != nil { return } - _, _ = w.client.Do(req) + _, _ = w.getClient().Do(req) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go index d77b37c0c..26605d979 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go @@ -17,12 +17,15 @@ package remote import ( "bytes" "context" + "encoding/hex" "fmt" "io" "net/http" "net/url" "strings" + "github.com/google/go-containerregistry/internal/ipaddr" + "github.com/google/go-containerregistry/internal/limit" "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/internal/verify" "github.com/google/go-containerregistry/pkg/authn" @@ -40,8 +43,9 @@ const ( // fetcher implements methods for reading from a registry. type fetcher struct { - target resource - client *http.Client + target resource + client *http.Client + limiter *pullLimiter } func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) { @@ -69,10 +73,38 @@ func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, er } return &fetcher{ target: target, - client: &http.Client{Transport: tr}, + client: &http.Client{ + Transport: tr, + CheckRedirect: checkRedirectSSRF, + }, + limiter: o.limiter, }, nil } +// checkRedirectSSRF rejects HTTP redirects that cross from a public host to a +// private or link-local IP literal. This prevents a malicious registry from +// issuing a 302 to a cloud instance metadata service (e.g. 169.254.169.254) +// or another internal network address during blob or manifest downloads. +// +// Same-host redirects and redirects to non-IP hostnames (including DNS names +// that may resolve to private addresses) are allowed. The first redirect in +// the chain uses the original request URL as the "origin host" via +// req.Response.Request, falling back to req.URL when no prior response exists. +func checkRedirectSSRF(req *http.Request, via []*http.Request) error { + if len(via) == 0 || req.Response == nil { + return nil + } + origHost := via[0].URL.Hostname() + destHost := req.URL.Hostname() + if destHost == origHost { + return nil // same-host redirect is always allowed + } + if ipaddr.IsPrivateOrLinkLocal(destHost) { + return fmt.Errorf("SSRF protection: redirect from %q to private/link-local host %q denied", origHost, destHost) + } + return nil +} + func (f *fetcher) Do(req *http.Request) (*http.Response, error) { return f.client.Do(req) } @@ -136,15 +168,31 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta return nil, nil, err } - manifest, err := io.ReadAll(io.LimitReader(resp.Body, manifestLimit)) + manifest, err := limit.ReadAll(resp.Body, manifestLimit) if err != nil { return nil, nil, err } - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + // Hash with the algorithm of the reference when pulling by digest. + dgst, byDigest := ref.(name.Digest) + algo := "sha256" + if byDigest { + h, err := v1.NewHash(dgst.DigestStr()) + if err != nil { + return nil, nil, err + } + algo = h.Algorithm + } + hasher, err := v1.Hasher(algo) if err != nil { return nil, nil, err } + hasher.Write(manifest) + digest := v1.Hash{ + Algorithm: algo, + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + } + size := int64(len(manifest)) mediaType := types.MediaType(resp.Header.Get("Content-Type")) contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) @@ -155,18 +203,27 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta } // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := ref.(name.Digest); ok { + if byDigest { if digest.String() != dgst.DigestStr() { return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), ref) } } var artifactType string + var annotations map[string]string mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - artifactType = string(mf.Config.MediaType) + if mf != nil { + // Per the OCI distribution spec, artifactType on the descriptor is + // set to the manifest's artifactType if present, otherwise it falls + // back to the config descriptor's mediaType. + if mf.ArtifactType != "" { + artifactType = mf.ArtifactType + } else { + artifactType = string(mf.Config.MediaType) + } + annotations = mf.Annotations } // Do nothing for tags; I give up. @@ -183,6 +240,7 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta Size: size, MediaType: mediaType, ArtifactType: artifactType, + Annotations: annotations, } return manifest, &desc, nil @@ -247,18 +305,30 @@ func (f *fetcher) headManifest(ctx context.Context, ref name.Reference, acceptab func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) { u := f.url("blobs", h.String()) + return f.fetchBlobURL(ctx, u, size, h) +} + +func (f *fetcher) fetchBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash) (io.ReadCloser, error) { + release, err := f.limiter.acquire(ctx) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { + release() return nil, err } resp, err := f.client.Do(req.WithContext(ctx)) if err != nil { + release() return nil, redact.Error(err) } if err := transport.CheckError(resp, http.StatusOK); err != nil { resp.Body.Close() + release() return nil, err } @@ -269,11 +339,22 @@ func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.Read if size == verify.SizeUnknown { size = hsize } else if hsize != size { + resp.Body.Close() + release() return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size) } } - return verify.ReadCloser(resp.Body, size, h) + rc, err := verify.ReadCloser(resp.Body, size, h) + if err != nil { + resp.Body.Close() + release() + return nil, err + } + return &limitedReadCloser{ + ReadCloser: rc, + release: release, + }, nil } func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, error) { @@ -296,6 +377,55 @@ func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, erro return resp, nil } +// validateForeignURL rejects foreign layer URLs that use a disallowed scheme +// or resolve to a private / link-local IP address (SSRF protection). DNS-based +// SSRF is out of scope, matching transport.validateRealmURL. +func validateForeignURL(rawURL string, insecure bool) error { + u, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("parsing foreign layer URL %q: %w", rawURL, err) + } + switch u.Scheme { + case "https": + case "http": + if !insecure { + return fmt.Errorf("foreign layer URL scheme %q not allowed for a secure registry; use https", u.Scheme) + } + default: + return fmt.Errorf("foreign layer URL scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) + } + host := u.Hostname() + if ipaddr.IsPrivateOrLinkLocal(host) { + return fmt.Errorf("foreign layer URL host %q is a private or link-local address", host) + } + return nil +} + +// fetchForeignBlobURL fetches a foreign-layer blob, validating every redirect +// destination through validateForeignURL (SSRF protection). +func (f *fetcher) fetchForeignBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash, insecure bool) (io.ReadCloser, error) { + safeClient := &http.Client{ + Transport: f.client.Transport, + CheckRedirect: func(req *http.Request, _ []*http.Request) error { + return validateForeignURL(req.URL.String(), insecure) + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := safeClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: unexpected status %s", u.String(), resp.Status) + } + return verify.ReadCloser(resp.Body, size, h) +} + func (f *fetcher) blobExists(ctx context.Context, h v1.Hash) (bool, error) { u := f.url("blobs", h.String()) req, err := http.NewRequest(http.MethodHead, u.String(), nil) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index f085967ed..d39a0e71f 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "io" - "net/http" "net/url" "sync" @@ -27,7 +26,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -172,7 +170,7 @@ func (rl *remoteImageLayer) Digest() (v1.Hash, error) { // Compressed implements partial.CompressedLayer func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { - urls := []url.URL{rl.ri.fetcher.url("blobs", rl.digest.String())} + u := rl.ri.fetcher.url("blobs", rl.digest.String()) // Add alternative layer sources from URLs (usually none). d, err := partial.BlobDescriptor(rl, rl.digest) @@ -187,38 +185,38 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { // We don't want to log binary layers -- this can break terminals. ctx := redact.NewContext(rl.ctx, "omitting binary blobs from logs") - for _, s := range d.URLs { - u, err := url.Parse(s) - if err != nil { - return nil, err - } - urls = append(urls, *u) - } + insecure := rl.ri.fetcher.target.Scheme() == "http" // The lastErr for most pulls will be the same (the first error), but for // foreign layers we'll want to surface the last one, since we try to pull // from the registry first, which would often fail. // TODO: Maybe we don't want to try pulling from the registry first? var lastErr error - for _, u := range urls { - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { + rc, err := rl.ri.fetcher.fetchBlobURL(ctx, u, d.Size, rl.digest) + if err == nil { + return rc, nil + } + lastErr = err + + foreignURLs := make([]url.URL, 0, len(d.URLs)) + for _, s := range d.URLs { + if err := validateForeignURL(s, insecure); err != nil { return nil, err } - - resp, err := rl.ri.fetcher.Do(req.WithContext(ctx)) + fu, err := url.Parse(s) if err != nil { - lastErr = err - continue + return nil, err } + foreignURLs = append(foreignURLs, *fu) + } - if err := transport.CheckError(resp, http.StatusOK); err != nil { - resp.Body.Close() + for _, fu := range foreignURLs { + rc, err := rl.ri.fetcher.fetchForeignBlobURL(ctx, fu, d.Size, rl.digest, insecure) + if err != nil { lastErr = err continue } - - return verify.ReadCloser(resp.Body, d.Size, rl.digest) + return rc, nil } return nil, lastErr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index b80972c80..3956a38e6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -225,8 +225,12 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - child.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + child.ArtifactType = mf.ArtifactType + } else { + child.ArtifactType = string(mf.Config.MediaType) + } } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go new file mode 100644 index 000000000..153d7cf02 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go @@ -0,0 +1,63 @@ +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "io" + "sync" +) + +type pullLimiter struct { + tokens chan struct{} +} + +func newPullLimiter(jobs int) *pullLimiter { + return &pullLimiter{ + tokens: make(chan struct{}, jobs), + } +} + +func (l *pullLimiter) acquire(ctx context.Context) (func(), error) { + if l == nil { + return func() {}, nil + } + select { + case l.tokens <- struct{}{}: + return func() { <-l.tokens }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +type limitedReadCloser struct { + io.ReadCloser + release func() + once sync.Once +} + +func (l *limitedReadCloser) Read(p []byte) (int, error) { + n, err := l.ReadCloser.Read(p) + if err != nil { + l.once.Do(l.release) + } + return n, err +} + +func (l *limitedReadCloser) Close() error { + err := l.ReadCloser.Close() + l.once.Do(l.release) + return err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go index 910d2a94c..9beff7dc2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go @@ -17,6 +17,7 @@ package remote import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -85,7 +86,7 @@ func (f *fetcher) listPage(ctx context.Context, repo name.Repository, next strin return nil, err } - uri, err := getNextPageURL(resp) + uri, err := getNextPageURL(resp, repo) if err != nil { return nil, err } @@ -99,8 +100,9 @@ func (f *fetcher) listPage(ctx context.Context, repo name.Repository, next strin // getNextPageURL checks if there is a Link header in a http.Response which // contains a link to the next page. If yes it returns the url.URL of the next -// page otherwise it returns nil. -func getNextPageURL(resp *http.Response) (*url.URL, error) { +// page otherwise it returns nil. It validates that the resolved URL points +// to the same registry to prevent SSRF attacks. +func getNextPageURL(resp *http.Response, repo name.Repository) (*url.URL, error) { link := resp.Header.Get("Link") if link == "" { return nil, nil @@ -115,6 +117,9 @@ func getNextPageURL(resp *http.Response) (*url.URL, error) { return nil, fmt.Errorf("failed to parse link header: missing '>' in: %s", link) } link = link[1:end] + if link == "" { + return nil, nil + } linkURL, err := url.Parse(link) if err != nil { @@ -124,9 +129,31 @@ func getNextPageURL(resp *http.Response) (*url.URL, error) { return nil, nil } linkURL = resp.Request.URL.ResolveReference(linkURL) + + // Validate that the pagination URL points to the same registry to prevent SSRF. + if err := validatePaginationURL(linkURL, repo); err != nil { + return nil, err + } + return linkURL, nil } +// validatePaginationURL checks that a pagination URL is safe to follow. +func validatePaginationURL(u *url.URL, repo name.Repository) error { + return validatePaginationURLHost(u, repo.Scheme(), repo.RegistryStr()) +} + +// validatePaginationURLHost checks that a pagination URL is safe to follow. +func validatePaginationURLHost(u *url.URL, scheme, host string) error { + if u.Scheme != scheme { + return fmt.Errorf("pagination URL scheme %q does not match registry scheme %q", u.Scheme, scheme) + } + if u.Host != host { + return errors.New("pagination URL host does not match registry host: potential SSRF attack") + } + return nil +} + type Lister struct { f *fetcher repo name.Repository @@ -150,3 +177,40 @@ func (l *Lister) Next(ctx context.Context) (*Tags, error) { func (l *Lister) HasNext() bool { return l.page != nil && (!l.needMore || l.page.Next != "") } + +// getNextPageURLForRegistry is like getNextPageURL but for name.Registry. +func getNextPageURLForRegistry(resp *http.Response, reg name.Registry) (*url.URL, error) { + link := resp.Header.Get("Link") + if link == "" { + return nil, nil + } + + if link[0] != '<' { + return nil, fmt.Errorf("failed to parse link header: missing '<' in: %s", link) + } + + end := strings.Index(link, ">") + if end == -1 { + return nil, fmt.Errorf("failed to parse link header: missing '>' in: %s", link) + } + link = link[1:end] + if link == "" { + return nil, nil + } + + linkURL, err := url.Parse(link) + if err != nil { + return nil, err + } + if resp.Request == nil || resp.Request.URL == nil { + return nil, nil + } + linkURL = resp.Request.URL.ResolveReference(linkURL) + + // Validate that the pagination URL points to the same registry to prevent SSRF. + if err := validatePaginationURLHost(linkURL, reg.Scheme(), reg.RegistryStr()); err != nil { + return nil, err + } + + return linkURL, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 15b7da1e4..749db0b68 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -45,6 +45,8 @@ type options struct { retryBackoff Backoff retryPredicate retry.Predicate retryStatusCodes []int + limiter *pullLimiter + referrersTagFallback bool // Only these options can overwrite Reuse()d options. platform v1.Platform @@ -92,6 +94,7 @@ var fastBackoff = Backoff{ var defaultRetryStatusCodes = []int{ http.StatusRequestTimeout, + http.StatusTooManyRequests, // 429: OCI distribution-spec rate limit; TooManyRequestsErrorCode is already classified temporary in transport/error.go http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, @@ -106,6 +109,8 @@ const ( // ECR returns an error if n > 1000: // https://github.com/google/go-containerregistry/issues/1091 defaultPageSize = 1000 + + defaultReferrersTagFallback = true ) // DefaultTransport is based on http.DefaultTransport with modifications @@ -127,14 +132,15 @@ var DefaultTransport http.RoundTripper = &http.Transport{ func makeOptions(opts ...Option) (*options, error) { o := &options{ - transport: DefaultTransport, - platform: defaultPlatform, - context: context.Background(), - jobs: defaultJobs, - pageSize: defaultPageSize, - retryPredicate: defaultRetryPredicate, - retryBackoff: defaultRetryBackoff, - retryStatusCodes: defaultRetryStatusCodes, + transport: DefaultTransport, + platform: defaultPlatform, + context: context.Background(), + jobs: defaultJobs, + pageSize: defaultPageSize, + retryPredicate: defaultRetryPredicate, + retryBackoff: defaultRetryBackoff, + retryStatusCodes: defaultRetryStatusCodes, + referrersTagFallback: defaultReferrersTagFallback, } for _, option := range opts { @@ -142,6 +148,7 @@ func makeOptions(opts ...Option) (*options, error) { return nil, err } } + o.limiter = newPullLimiter(o.jobs) switch { case o.auth != nil && o.keychain != nil: @@ -325,6 +332,24 @@ func WithRetryStatusCodes(codes ...int) Option { } } +// WithReferrersTagFallback toggles the referrers tag fallback scheme, see: +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#referrers-tag-schema +// +// When enabled, pushing a manifest with a subject to a registry that doesn't +// support the Referrers API maintains an image index of referrers under a +// fallback tag, and listing referrers reads from that tag. When disabled, +// the Referrers API is required: pushing a manifest with a subject and +// listing referrers both fail against a registry that doesn't support it, +// rather than falling back to the tag scheme. +// +// The default is true. +func WithReferrersTagFallback(enabled bool) Option { + return func(o *options) error { + o.referrersTagFallback = enabled + return nil + } +} + // WithFilter sets the filter querystring for HTTP operations. func WithFilter(key string, value string) Option { return func(o *options) error { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go index 7da8017ee..094162ec4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go @@ -76,7 +76,11 @@ func (p *Puller) fetcher(ctx context.Context, target resource) (*fetcher, error) o: p.o, }) rr := v.(*reader) - return rr.f, rr.init(ctx) + if err := rr.init(ctx); err != nil { + p.readers.CompareAndDelete(target, rr) + return nil, err + } + return rr.f, nil } // Head is like remote.Head, but avoids re-authenticating when possible. @@ -218,5 +222,5 @@ func (p *Puller) referrers(ctx context.Context, d name.Digest, filter map[string if err != nil { return nil, err } - return f.fetchReferrers(ctx, filter, d) + return f.fetchReferrers(ctx, filter, d, p.o.referrersTagFallback) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go index 47e3b806e..3466c7a67 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go @@ -122,7 +122,11 @@ func (p *Pusher) writer(ctx context.Context, repo name.Repository, o *options) ( o: o, }) rw := v.(*repoWriter) - return rw, rw.init(ctx) + if err := rw.init(ctx); err != nil { + p.writers.CompareAndDelete(repo, rw) + return nil, err + } + return rw, nil } func (p *Pusher) Put(ctx context.Context, ref name.Reference, t Taggable) error { @@ -156,7 +160,11 @@ func (p *Pusher) Upload(ctx context.Context, repo name.Repository, l v1.Layer) e } func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { - w, err := p.writer(ctx, ref.Context(), p.o) + // Use a transport scoped for delete. Requesting DeleteScope (which + // includes the "delete" action) allows registries that require an + // explicit delete permission—such as IBM Cloud Container Registry—to + // grant access. + client, err := makeDeleteClient(ctx, ref.Context(), p.o) if err != nil { return err } @@ -172,7 +180,7 @@ func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { return err } - resp, err := w.w.client.Do(req.WithContext(ctx)) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return err } @@ -406,7 +414,7 @@ func (rw *repoWriter) writeChild(ctx context.Context, child partial.Describable, func (rw *repoWriter) manifestExists(ctx context.Context, ref name.Reference, t Taggable) (bool, error) { f := &fetcher{ target: ref.Context(), - client: rw.w.client, + client: rw.w.getClient(), } m, err := taggableToManifest(t) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go index 4bc6f70a8..5a111d7fc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go @@ -18,10 +18,11 @@ import ( "bytes" "context" "errors" - "io" + "fmt" "net/http" "strings" + "github.com/google/go-containerregistry/internal/limit" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" @@ -46,7 +47,7 @@ func fallbackTag(d name.Digest) name.Tag { return d.Context().Tag(strings.Replace(d.DigestStr(), ":", "-", 1)) } -func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, d name.Digest) (v1.ImageIndex, error) { +func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, d name.Digest, tagFallback bool) (v1.ImageIndex, error) { // Check the Referrers API endpoint first. u := f.url("referrers", d.DigestStr()) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) @@ -67,11 +68,14 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, var b []byte if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Type") == string(types.OCIImageIndex) { - b, err = io.ReadAll(resp.Body) + b, err = limit.ReadAll(resp.Body, manifestLimit) if err != nil { return nil, err } } else { + if !tagFallback { + return nil, fmt.Errorf("registry %s does not support the Referrers API and the referrers tag fallback is disabled", d.Context().RegistryStr()) + } // The registry doesn't support the Referrers API endpoint, so we'll use the fallback tag scheme. b, _, err = f.fetchManifest(ctx, fallbackTag(d), []types.MediaType{types.OCIImageIndex}) var terr *transport.Error diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index 44a323ee3..b98482763 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -19,13 +19,14 @@ import ( "encoding/json" "errors" "fmt" - "io" "net" "net/http" "net/url" "strings" "sync" + "github.com/google/go-containerregistry/internal/ipaddr" + "github.com/google/go-containerregistry/internal/limit" "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/logs" @@ -33,6 +34,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge" ) +// maxTokenBodySize limits bearer token response body reads to prevent OOM +// when a token endpoint returns an unexpectedly large body. +const maxTokenBodySize = 64 * 1024 // 64 KiB + type Token struct { Token string `json:"token"` AccessToken string `json:"access_token,omitempty"` @@ -86,7 +91,7 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip // registry can supply a realm pointing at an internal service or cloud // metadata endpoint (e.g. 169.254.169.254), causing SSRF when the client // subsequently fetches a token. - if err := validateRealmURL(realm, pr.Insecure); err != nil { + if err := validateRealmURL(realm, reg.RegistryStr(), pr.Insecure); err != nil { return nil, fmt.Errorf("invalid realm in www-authenticate: %w", err) } service := pr.Parameters["service"] @@ -105,10 +110,24 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip }, nil } +// realmRedirectCheck mimics the default http.Client redirect policy but also +// validates each redirect URL with validateRealmURL. +func realmRedirectCheck(registryHost string, insecure bool) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + if err := validateRealmURL(req.URL.String(), registryHost, insecure); err != nil { + return fmt.Errorf("refusing token-server redirect to %q: %w", req.URL, err) + } + return nil + } +} + // validateRealmURL returns an error if the realm URL uses a disallowed scheme -// or resolves to a private / link-local IP address. This prevents a crafted -// WWW-Authenticate header from redirecting token fetches to internal services. -func validateRealmURL(realm string, insecure bool) error { +// or resolves to a private / link-local IP address. Realm URLs matching the +// registry host:port are always allowed. See #2258. +func validateRealmURL(realm, registryHost string, insecure bool) error { u, err := url.Parse(realm) if err != nil { return fmt.Errorf("parsing realm %q: %w", realm, err) @@ -123,16 +142,18 @@ func validateRealmURL(realm string, insecure bool) error { default: return fmt.Errorf("realm scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) } + // Always allow realms matching the registry host:port. + if registryHost != "" && u.Host == registryHost { + return nil + } // Reject IP literals that resolve to private or link-local ranges. // This blocks direct references to RFC 1918 addresses, loopback, and // link-local ranges including the cloud instance metadata service // (169.254.169.254 / fd00:ec2::254). DNS-based SSRF is out of scope // here; callers should apply network-level controls if needed. host := u.Hostname() - if ip := net.ParseIP(host); ip != nil { - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() { - return fmt.Errorf("realm host %q is a private or link-local address", host) - } + if ipaddr.IsPrivateOrLinkLocal(host) { + return fmt.Errorf("realm host %q is a private or link-local address", host) } return nil } @@ -199,6 +220,16 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // close out old response, since we will not return it. res.Body.Close() + // For cross-host challenges (the request was redirected to another host), + // never mutate bt's shared state: accumulating this host's scope into + // bt.scopes or refreshing bt's token from bt.realm would pollute future + // same-host requests with a scope/token that belongs to a request we + // only ever intended to send to the redirected host. Instead, attempt a + // fresh per-host token exchange scoped entirely to this one request. + if !matchesHost(bt.registry.RegistryStr(), in, bt.scheme) { + return bt.handleCrossHostChallenge(in, challenges) + } + newScopes := []string{} bt.mx.Lock() got := stringSet(bt.scopes) @@ -225,12 +256,76 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { if err = bt.refresh(in.Context()); err != nil { return nil, err } - return sendRequest() + + bt.mx.RLock() + tok := bt.bearer.RegistryToken + bt.mx.RUnlock() + in.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tok)) + return bt.inner.RoundTrip(in) } return res, err } +// handleCrossHostChallenge performs a per-host bearer token exchange when +// the request has been redirected to a host different from bt.registry. It +// parses the redirected host's own WWW-Authenticate Bearer challenge, fetches +// a token from that host's realm using anonymous auth (the original registry's +// credentials are never forwarded cross-host), and applies the token only to +// this request. +// +// If no usable Bearer challenge is present, or the token exchange fails, the +// request is retried without an Authorization header. +func (bt *bearerTransport) handleCrossHostChallenge(in *http.Request, challenges []authchallenge.Challenge) (*http.Response, error) { + for _, wac := range challenges { + if strings.ToLower(wac.Scheme) != "bearer" { + continue + } + if _, ok := wac.Parameters["realm"]; !ok { + continue + } + + redirectedReg, err := name.NewRegistry(in.URL.Host, name.WeakValidation) + if err != nil { + logs.Warn.Printf("cross-host redirect: invalid host %q: %v", in.URL.Host, err) + continue + } + + // Build a transport.Challenge from the redirected host's WWW-Authenticate + // parameters. fromChallenge validates the realm URL (SSRF guard). + pr := &Challenge{ + Scheme: wac.Scheme, + Parameters: wac.Parameters, + Insecure: in.URL.Scheme == "http", + } + scope := wac.Parameters["scope"] + // TODO: use a keychain to resolve credentials for the redirected host so + // that token endpoints requiring auth are also supported. For now, anonymous + // auth covers the common case where the redirected host's token endpoint is + // public. See https://github.com/google/go-containerregistry/issues/2359. + tmpBt, err := fromChallenge(redirectedReg, authn.Anonymous, bt.inner, pr, scope) + if err != nil { + logs.Warn.Printf("cross-host bearer challenge setup for %q failed: %v", in.URL.Host, err) + continue + } + + if err := tmpBt.refresh(in.Context()); err != nil { + logs.Warn.Printf("cross-host bearer exchange for %q failed: %v", in.URL.Host, err) + continue + } + + tmpBt.mx.RLock() + tok := tmpBt.bearer.RegistryToken + tmpBt.mx.RUnlock() + + in.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tok)) + return bt.inner.RoundTrip(in) + } + + // No usable bearer challenge; retry without credentials. + return bt.inner.RoundTrip(in) +} + // It's unclear which authentication flow to use based purely on the protocol, // so we rely on heuristics and fallbacks to support as many registries as possible. // The basic token exchange is attempted first, falling back to the oauth flow. @@ -374,7 +469,8 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { v.Set("access_type", "offline") } - client := http.Client{Transport: bt.inner} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: bt.inner, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return nil, err @@ -397,7 +493,7 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return limit.ReadAll(resp.Body, maxTokenBodySize) } // https://docs.docker.com/registry/spec/auth/token/ @@ -411,7 +507,8 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { auth: bt.basic, target: u.Host, } - client := http.Client{Transport: b} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: b, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} v := u.Query() bt.mx.RLock() @@ -441,5 +538,5 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return limit.ReadAll(resp.Body, maxTokenBodySize) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index d38e67624..7b14b8c41 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -15,12 +15,14 @@ package transport import ( + "bytes" "encoding/json" "fmt" "io" "net/http" "strings" + "github.com/google/go-containerregistry/internal/limit" "github.com/google/go-containerregistry/internal/redact" ) @@ -112,6 +114,10 @@ type ErrorCode string // The set of error conditions a registry may return: // https://github.com/distribution/distribution/blob/aac2f6c8b7c5a6c60190848bab5cbeed2b5ba0a9/docs/spec/api.md#errors-2 +// maxErrorBodySize limits HTTP error response body reads to prevent OOM when +// a registry returns an unexpectedly large error body. +const maxErrorBodySize = 64 * 1024 // 64 KiB + const ( BlobUnknownErrorCode ErrorCode = "BLOB_UNKNOWN" BlobUploadInvalidErrorCode ErrorCode = "BLOB_UPLOAD_INVALID" @@ -146,6 +152,7 @@ var temporaryErrorCodes = map[ErrorCode]struct{}{ var temporaryStatusCodes = map[int]struct{}{ http.StatusRequestTimeout: {}, + http.StatusTooManyRequests: {}, // matches TooManyRequestsErrorCode in temporaryErrorCodes http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, @@ -161,7 +168,7 @@ func CheckError(resp *http.Response, codes ...int) error { } } - b, err := io.ReadAll(resp.Body) + b, err := limit.ReadAll(resp.Body, maxErrorBodySize) if err != nil { return err } @@ -185,11 +192,16 @@ func makeError(resp *http.Response, body []byte) *Error { } func retryError(resp *http.Response) error { - b, err := io.ReadAll(resp.Body) + b, err := limit.ReadAll(resp.Body, maxErrorBodySize) if err != nil { return err } + // Restore the body so that a subsequent CheckError call (after the + // retry loop exhausts its retries) can still read and parse the + // structured registry error from the response. + resp.Body = io.NopCloser(bytes.NewReader(b)) + rerr := makeError(resp, b) rerr.temporary = true return rerr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go index c3b56f7a4..6ee7782b7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go @@ -18,7 +18,9 @@ package transport const ( PullScope string = "pull" PushScope string = "push,pull" - // For now DELETE is PUSH, which is the read/write ACL. - DeleteScope string = PushScope + // DeleteScope requests "delete" in addition to push/pull so that + // registries requiring an explicit delete action (e.g. IBM Cloud + // Container Registry) grant the necessary access. + DeleteScope string = "push,pull,delete" CatalogScope string = "catalog" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 58adfd8d3..4064711f4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -27,6 +27,7 @@ import ( "strings" "sync" + "github.com/google/go-containerregistry/internal/ipaddr" "github.com/google/go-containerregistry/internal/redact" "github.com/google/go-containerregistry/internal/retry" "github.com/google/go-containerregistry/pkg/authn" @@ -60,12 +61,40 @@ type writer struct { backoff Backoff predicate retry.Predicate + referrersTagFallback bool + scopeLock sync.Mutex // Keep track of scopes that we have already requested. scopeSet map[string]struct{} scopes []string } +// getClient returns the HTTP client, blocking on scope updates. +func (w *writer) getClient() *http.Client { + w.scopeLock.Lock() + defer w.scopeLock.Unlock() + return w.client +} + +// makeDeleteClient returns an HTTP client whose token includes the "delete" +// action so that registries requiring an explicit delete permission grant +// access for manifest deletion. +func makeDeleteClient(ctx context.Context, repo name.Repository, o *options) (*http.Client, error) { + auth := o.auth + if o.keychain != nil { + kauth, err := authn.Resolve(ctx, o.keychain, repo) + if err != nil { + return nil, err + } + auth = kauth + } + tr, err := transport.NewWithContext(ctx, repo.Registry, auth, o.transport, []string{repo.Scope(transport.DeleteScope)}) + if err != nil { + return nil, err + } + return &http.Client{Transport: tr, CheckRedirect: checkRedirectSSRF}, nil +} + func makeWriter(ctx context.Context, repo name.Repository, ls []v1.Layer, o *options) (*writer, error) { auth := o.auth if o.keychain != nil { @@ -86,15 +115,16 @@ func makeWriter(ctx context.Context, repo name.Repository, ls []v1.Layer, o *opt scopeSet[scope] = struct{}{} } return &writer{ - repo: repo, - client: &http.Client{Transport: tr}, - auth: auth, - transport: o.transport, - progress: o.progress, - backoff: o.retryBackoff, - predicate: o.retryPredicate, - scopes: scopes, - scopeSet: scopeSet, + repo: repo, + client: &http.Client{Transport: tr, CheckRedirect: checkRedirectSSRF}, + auth: auth, + transport: o.transport, + progress: o.progress, + backoff: o.retryBackoff, + predicate: o.retryPredicate, + referrersTagFallback: o.referrersTagFallback, + scopes: scopes, + scopeSet: scopeSet, }, nil } @@ -129,7 +159,7 @@ func (w *writer) maybeUpdateScopes(ctx context.Context, ml *MountableLayer) erro if err != nil { return err } - w.client = &http.Client{Transport: wt} + w.client = &http.Client{Transport: wt, CheckRedirect: checkRedirectSSRF} } return nil @@ -148,7 +178,27 @@ func (w *writer) nextLocation(resp *http.Response) (string, error) { // If the location header returned is just a url path, then fully qualify it. // We cannot simply call w.url, since there might be an embedded query string. - return resp.Request.URL.ResolveReference(u).String(), nil + resolved := resp.Request.URL.ResolveReference(u) + + // Reject Location headers that redirect to a DIFFERENT host that resolves to + // a private or link-local IP literal. A malicious or compromised registry can + // respond to a blob upload initiation (POST /v2/.../blobs/uploads/) with a + // crafted Location header pointing at an internal service, causing the client + // to send subsequent PATCH/PUT requests (including the layer data as the body) + // to that internal address. Pre-signed blob URLs from cloud storage providers + // (GCS, S3, Azure Blob) use public hostnames, so legitimate cross-host + // redirects are unaffected. + // + // Same-host redirects (e.g. a different path on the registry itself) are + // always allowed regardless of whether the registry IP is private. + origHost := resp.Request.URL.Hostname() + if destHost := resolved.Hostname(); destHost != origHost { + if ipaddr.IsPrivateOrLinkLocal(destHost) { + return "", fmt.Errorf("SSRF protection: Location header redirects to private/link-local host %q", destHost) + } + } + + return resolved.String(), nil } // checkExistingBlob checks if a blob exists already in the repository by making a @@ -163,7 +213,7 @@ func (w *writer) checkExistingBlob(ctx context.Context, h v1.Hash) (bool, error) return false, err } - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { return false, err } @@ -201,7 +251,7 @@ func (w *writer) initiateUpload(ctx context.Context, from, mount, origin string) return "", false, err } req.Header.Set("Content-Type", "application/json") - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { if from != "" { // https://github.com/google/go-containerregistry/issues/1679 @@ -281,7 +331,7 @@ func (w *writer) streamBlob(ctx context.Context, layer v1.Layer, streamLocation } req.Header.Set("Content-Type", "application/octet-stream") - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { return "", err } @@ -313,7 +363,7 @@ func (w *writer) commitBlob(ctx context.Context, location, digest string) error } req.Header.Set("Content-Type", "application/octet-stream") - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { return err } @@ -472,7 +522,7 @@ func (w *writer) commitSubjectReferrers(ctx context.Context, sub name.Digest, ad return err } req.Header.Set("Accept", string(types.OCIImageIndex)) - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { return err } @@ -485,6 +535,9 @@ func (w *writer) commitSubjectReferrers(ctx context.Context, sub name.Digest, ad // The registry supports Referrers API. The registry is responsible for updating the referrers list. return nil } + if !w.referrersTagFallback { + return fmt.Errorf("registry %s does not support the Referrers API and the referrers tag fallback is disabled", w.repo.RegistryStr()) + } // The registry doesn't support Referrers API, we need to update the manifest tagged with the fallback tag. // Make the request to GET the current manifest. @@ -495,7 +548,7 @@ func (w *writer) commitSubjectReferrers(ctx context.Context, sub name.Digest, ad return err } req.Header.Set("Accept", string(types.OCIImageIndex)) - resp, err = w.client.Do(req.WithContext(ctx)) + resp, err = w.getClient().Do(req.WithContext(ctx)) if err != nil { return err } @@ -555,9 +608,11 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } var mf struct { - MediaType types.MediaType `json:"mediaType"` - Subject *v1.Descriptor `json:"subject,omitempty"` - Config struct { + MediaType types.MediaType `json:"mediaType"` + Subject *v1.Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Config struct { MediaType types.MediaType `json:"mediaType"` } `json:"config"` } @@ -581,7 +636,7 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere } req.Header.Set("Content-Type", string(desc.MediaType)) - resp, err := w.client.Do(req.WithContext(ctx)) + resp, err := w.getClient().Do(req.WithContext(ctx)) if err != nil { return err } @@ -599,10 +654,15 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } desc := v1.Descriptor{ - ArtifactType: string(mf.Config.MediaType), - MediaType: mf.MediaType, - Digest: h, - Size: size, + MediaType: mf.MediaType, + Digest: h, + Size: size, + Annotations: mf.Annotations, + } + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) } if err := w.commitSubjectReferrers(ctx, ref.Context().Digest(mf.Subject.Digest.String()), diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index 96c0cce49..76b61c687 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -248,7 +248,7 @@ func followLinks(opener Opener, filePath string, visited map[string]bool) (io.Re if err != nil { return nil, err } - if hdr.Name == filePath { + if path.Clean(hdr.Name) == path.Clean(filePath) { if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink { currentDir := filepath.Dir(filePath) return followLinks(opener, path.Join(currentDir, path.Clean(hdr.Linkname)), visited) @@ -304,6 +304,9 @@ func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, } for idx, diffID := range cfg.RootFS.DiffIDs { if diffID == h { + if idx >= len(i.imgDescriptor.Layers) { + return nil, fmt.Errorf("config has %d rootfs.diff_id(s) but tarball manifest only references %d layer(s); the config may not describe a runnable image", len(cfg.RootFS.DiffIDs), len(i.imgDescriptor.Layers)) + } // Technically the media type should be 'application/tar' but given that our // v1.Layer doesn't force consumers to care about whether the layer is compressed // we should be fine returning the DockerLayer media type @@ -374,6 +377,9 @@ func (c *compressedImage) Manifest() (*v1.Manifest, error) { if err != nil { return nil, err } + if i >= len(cfg.RootFS.DiffIDs) { + return nil, fmt.Errorf("tarball manifest references %d layer(s) but config has %d rootfs.diff_ids; the config may not describe a runnable image (for example, a buildkit cacheconfig)", len(c.imgDescriptor.Layers), len(cfg.RootFS.DiffIDs)) + } diffid := cfg.RootFS.DiffIDs[i] if d, ok := c.imgDescriptor.LayerSources[diffid]; ok { // If it's a foreign layer, just append the descriptor so we can avoid diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 8a2630961..5e63d52d5 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -22,10 +22,7 @@ import ( "os" "sync" - "github.com/containerd/stargz-snapshotter/estargz" - "github.com/google/go-containerregistry/internal/and" comp "github.com/google/go-containerregistry/internal/compression" - gestargz "github.com/google/go-containerregistry/internal/estargz" ggzip "github.com/google/go-containerregistry/internal/gzip" "github.com/google/go-containerregistry/internal/zstd" "github.com/google/go-containerregistry/pkg/compression" @@ -43,7 +40,6 @@ type layer struct { compression compression.Compression compressionLevel int annotations map[string]string - estgzopts []estargz.Option mediaType types.MediaType } @@ -161,53 +157,15 @@ func WithCompressedCaching(l *layer) { // through estargz.Options to the underlying compression layer. This is // only meaningful when estargz is enabled. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargzOptions(opts ...estargz.Option) LayerOption { - return func(l *layer) { - l.estgzopts = opts - } +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargzOptions(...any) LayerOption { + return func(*layer) {} } // WithEstargz is a functional option that explicitly enables estargz support. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargz(l *layer) { - oguncompressed := l.uncompressedopener - estargz := func() (io.ReadCloser, error) { - crc, err := oguncompressed() - if err != nil { - return nil, err - } - eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compressionLevel)) - rc, h, err := gestargz.ReadCloser(crc, eopts...) - if err != nil { - return nil, err - } - l.annotations[estargz.TOCJSONDigestAnnotation] = h.String() - return &and.ReadCloser{ - Reader: rc, - CloseFunc: func() error { - err := rc.Close() - if err != nil { - return err - } - // As an optimization, leverage the DiffID exposed by the estargz ReadCloser - l.diffID, err = v1.NewHash(rc.DiffID().String()) - return err - }, - }, nil - } - uncompressed := func() (io.ReadCloser, error) { - urc, err := estargz() - if err != nil { - return nil, err - } - return ggzip.UnzipReadCloser(urc) - } - - l.compressedopener = estargz - l.uncompressedopener = uncompressed -} +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargz(*layer) {} // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) { @@ -241,11 +199,6 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { mediaType: types.DockerLayer, } - if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" { - logs.Warn.Println("GGCR_EXPERIMENT_ESTARGZ is deprecated, and will be removed in a future release.") - opts = append([]LayerOption{WithEstargz}, opts...) - } - switch comp { case compression.GZip: layer.compressedopener = opener diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go index e607df164..4b1bfd0a6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go @@ -28,6 +28,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" ) // WriteToFile writes in the compressed format to a tarball, on disk. @@ -184,23 +185,14 @@ func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w // gunzip expects certain file extensions: // https://www.gnu.org/software/gzip/manual/html_node/Overview.html - layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + layerFiles[i] = hex + layerExtension(l) if _, ok := seenLayerDigests[hex]; ok { continue } seenLayerDigests[hex] = struct{}{} - r, err := l.Compressed() - if err != nil { - return sendProgressWriterReturn(pw, err) - } - blobSize, err := l.Size() - if err != nil { - return sendProgressWriterReturn(pw, err) - } - - if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil { + if err := writeLayer(tf, layerFiles[i], l); err != nil { return sendProgressWriterReturn(pw, err) } } @@ -253,7 +245,7 @@ func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error // gunzip expects certain file extensions: // https://www.gnu.org/software/gzip/manual/html_node/Overview.html - layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + layerFiles[i] = hex + layerExtension(l) // Add to LayerSources if it's a foreign layer. desc, err := partial.BlobDescriptor(img, d) @@ -380,6 +372,23 @@ func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error { return err } +// writeLayer streams a layer's compressed blob into the tar writer and closes +// the reader before returning. The close releases any pull-limiter slot held +// by a remote-backed layer (remote.WithJobs); leaving it open would deadlock +// the write loop after defaultJobs layers. +func writeLayer(tf *tar.Writer, name string, l v1.Layer) error { + r, err := l.Compressed() + if err != nil { + return err + } + defer r.Close() + blobSize, err := l.Size() + if err != nil { + return err + } + return writeTarEntry(tf, name, r, blobSize) +} + // ComputeManifest get the manifest.json that will be written to the tarball // for multiple references func ComputeManifest(refToImage map[name.Reference]v1.Image) (Manifest, error) { @@ -455,3 +464,16 @@ func calculateSingleFileInTarSize(in int64) (out int64) { out += 512 return out } + +func layerExtension(l v1.Layer) string { + mt, _ := l.MediaType() + switch mt { + case types.OCILayerZStd: + return ".tar.zst" + case types.OCIUncompressedLayer, types.OCIUncompressedRestrictedLayer, types.DockerUncompressedLayer: + return ".tar" + default: + // historically, only gzip compressed tar was supported, so this is the fallback + return ".tar.gz" + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/types/types.go b/vendor/github.com/google/go-containerregistry/pkg/v1/types/types.go index c86657d7b..69878980b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/types/types.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/types/types.go @@ -29,6 +29,7 @@ const ( OCIRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" OCIUncompressedLayer MediaType = "application/vnd.oci.image.layer.v1.tar" OCIUncompressedRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar" + OCIEmptyJSON MediaType = "application/vnd.oci.empty.v1+json" DockerManifestSchema1 MediaType = "application/vnd.docker.distribution.manifest.v1+json" DockerManifestSchema1Signed MediaType = "application/vnd.docker.distribution.manifest.v1+prettyjws" diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go index 4e684c7de..69edf5eff 100644 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go +++ b/vendor/github.com/grpc-ecosystem/grpc-gateway/v2/runtime/mux.go @@ -71,6 +71,7 @@ type ServeMux struct { streamErrorHandler StreamErrorHandlerFunc routingErrorHandler RoutingErrorHandlerFunc disablePathLengthFallback bool + disableHTTPMethodOverride bool unescapingMode UnescapingMode writeContentLength bool disableChunkedEncoding bool @@ -271,6 +272,19 @@ func WithDisablePathLengthFallback() ServeMuxOption { } } +// WithDisableHTTPMethodOverride returns a ServeMuxOption that disables the +// X-HTTP-Method-Override header handling. +// +// When this option is used, the mux will no longer allow POST requests with +// the X-HTTP-Method-Override header to override the HTTP method. The path +// length fallback (POST with application/x-www-form-urlencoded falling back +// to a matching GET handler) is not affected by this option. +func WithDisableHTTPMethodOverride() ServeMuxOption { + return func(serveMux *ServeMux) { + serveMux.disableHTTPMethodOverride = true + } +} + // WithWriteContentLength returns a ServeMuxOption to enable writing content length on non-streaming responses func WithWriteContentLength() ServeMuxOption { return func(serveMux *ServeMux) { @@ -405,7 +419,7 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { path = r.URL.RawPath } - if override := r.Header.Get("X-HTTP-Method-Override"); override != "" && s.isPathLengthFallback(r) { + if override := r.Header.Get("X-HTTP-Method-Override"); override != "" && !s.disableHTTPMethodOverride && s.isPathLengthFallback(r) { if err := r.ParseForm(); err != nil { _, outboundMarshaler := MarshalerForRequest(s, r) sterr := status.Error(codes.InvalidArgument, err.Error()) @@ -467,6 +481,7 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { HTTPStatus: http.StatusBadRequest, Err: mse, }) + return } continue } @@ -509,6 +524,7 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { HTTPStatus: http.StatusBadRequest, Err: mse, }) + return } continue } diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes index 402433593..57aa6487c 100644 --- a/vendor/github.com/klauspost/compress/.gitattributes +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -1,2 +1,3 @@ * -text *.bin -text -diff +*.md text eol=lf diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index e839fe9c6..0e9f170d0 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -1,700 +1,712 @@ -# compress - -This package provides various compression algorithms. - -* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. -* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). -* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. -* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. -* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. -* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. - -[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) -[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) -[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) - -# package usage - -Use `go get github.com/klauspost/compress@latest` to add it to your project. - -This package will support the current Go version and 2 versions back. - -* Use the `nounsafe` tag to disable all use of the "unsafe" package. -* Use the `noasm` tag to disable all assembly across packages. - -Use the links above for more information on each. - -# changelog - -* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) - * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 - * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 - * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 - -* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) - * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). - -* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) - * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 - * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 - -* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED - * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 - * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 - * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 - * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 - * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 - * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 - * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 - * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 - -* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) - * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 - * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 - * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 - * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 - * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 - * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 - * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 - -
- See changes to v1.17.x - -* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) - * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 - * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 - * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 - * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 - -* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) - * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 - * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 - * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 - * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 - * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 - -* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) - * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 - * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 - * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 - * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 - -* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) - * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 - * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 - -* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) - * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 - * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 - -* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) - * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 - * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 - -* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) - * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 - * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 - * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 - * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 - * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 -https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 - -* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) - * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 - * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 - * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 - * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 - * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 - -* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) - * fse: Fix max header size https://github.com/klauspost/compress/pull/881 - * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 - * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 - -* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) - * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 - -* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) - * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 - * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 - * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 - -* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) - * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 - * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 - * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 - * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 - * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 - * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 - -
-
- See changes to v1.16.x - - -* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) - * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 - * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 - -* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) - * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 - * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 - * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 - * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 - -* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) - * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 - * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 - -* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) - * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 - * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 - * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 - * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 - * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 - * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 - -* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) - * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 - * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 - * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 - * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 - * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 - -* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) - * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 - * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 - * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 - * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 - * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 - * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 -
- -
- See changes to v1.15.x - -* Jan 21st, 2023 (v1.15.15) - * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 - * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 - * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 - * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 - -* Jan 3rd, 2023 (v1.15.14) - - * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 - * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 - * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 - * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 - -* Dec 11, 2022 (v1.15.13) - * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 - * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 - -* Oct 26, 2022 (v1.15.12) - - * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 - * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 - -* Sept 26, 2022 (v1.15.11) - - * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 - * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 - * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 - * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 - -* Sept 16, 2022 (v1.15.10) - - * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 - * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 - * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 - * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 - * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 - * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 - * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 - -* July 21, 2022 (v1.15.9) - - * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 - * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 - * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 - -* July 13, 2022 (v1.15.8) - - * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 - * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 - * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 - * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 - * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 - * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 - * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 - -* June 29, 2022 (v1.15.7) - - * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 - * zip: Merge upstream https://github.com/klauspost/compress/pull/631 - * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 - * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 - * flate: Faster histograms https://github.com/klauspost/compress/pull/620 - * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 - -* June 3, 2022 (v1.15.6) - * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 - * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 - * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 - * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 - * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 - * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 - * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 - * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 - * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 - * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 - -* May 25, 2022 (v1.15.5) - * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 - * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 - * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 - * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 - * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 - * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 - * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 - * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 - * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 - - -* May 11, 2022 (v1.15.4) - * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) - * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) - * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) - * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) - -* May 5, 2022 (v1.15.3) - * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) - * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) - -* Apr 26, 2022 (v1.15.2) - * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) - * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) - * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) - * Minimum version is Go 1.16, added CI test on 1.18. - -* Mar 11, 2022 (v1.15.1) - * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) - * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) - * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) - * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) - * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) - -* Mar 3, 2022 (v1.15.0) - * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) - * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) - * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) - * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) - * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) - * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) - -Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. - -Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. - -While the release has been extensively tested, it is recommended to testing when upgrading. - -
- -
- See changes to v1.14.x - -* Feb 22, 2022 (v1.14.4) - * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) - * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) - * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 - * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) - -* Feb 17, 2022 (v1.14.3) - * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) - * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) - * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) - -* Jan 25, 2022 (v1.14.2) - * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) - * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) - * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) - * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) - * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) - * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) - -* Jan 11, 2022 (v1.14.1) - * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) - * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) - * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) - * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) - * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) -
- -
- See changes to v1.13.x - -* Aug 30, 2021 (v1.13.5) - * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) - * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) - * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) - * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) - -* Aug 12, 2021 (v1.13.4) - * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). - * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) - -* Aug 3, 2021 (v1.13.3) - * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) - * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) - * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) - * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) - * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) - * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) - -* Jun 14, 2021 (v1.13.1) - * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) - * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) - * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) - * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) - -* Jun 3, 2021 (v1.13.0) - * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. - * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) - * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) -
- - -
- See changes to v1.12.x - -* May 25, 2021 (v1.12.3) - * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) - * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) - * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) - -* Apr 27, 2021 (v1.12.2) - * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) - * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) - * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) - * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) - * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) - * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) - -* Apr 14, 2021 (v1.12.1) - * snappy package removed. Upstream added as dependency. - * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) - * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) - * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) - * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) - * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) - * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) -
- -
- See changes to v1.11.x - -* Mar 26, 2021 (v1.11.13) - * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) - * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) - * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) - * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) - -* Mar 5, 2021 (v1.11.12) - * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). - * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) - -* Mar 1, 2021 (v1.11.9) - * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) - * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) - * s2: Fix binaries. - -* Feb 25, 2021 (v1.11.8) - * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. - * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) - * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) - * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) - * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) - -* Jan 14, 2021 (v1.11.7) - * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) - * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) - * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) - * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) - * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) - -* Jan 7, 2021 (v1.11.6) - * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) - * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) - -* Dec 20, 2020 (v1.11.4) - * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) - * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) - * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) - * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) - * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) - -* Nov 15, 2020 (v1.11.3) - * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) - * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) - -* Oct 11, 2020 (v1.11.2) - * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) - -* Oct 1, 2020 (v1.11.1) - * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) - -* Sept 8, 2020 (v1.11.0) - * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) - * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) - * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) -
- -
- See changes to v1.10.x - -* July 8, 2020 (v1.10.11) - * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) - * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) - -* June 23, 2020 (v1.10.10) - * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) - -* June 16, 2020 (v1.10.9): - * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) - * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) - * Fuzzit tests removed. The service has been purchased and is no longer available. - -* June 5, 2020 (v1.10.8): - * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) - -* June 1, 2020 (v1.10.7): - * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) - * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) - * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) - -* May 21, 2020: (v1.10.6) - * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) - * zstd: Stricter decompression checks. - -* April 12, 2020: (v1.10.5) - * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) - -* Apr 8, 2020: (v1.10.4) - * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) -* Mar 11, 2020: (v1.10.3) - * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) - * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) - * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) - * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) - * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) - -* Feb 27, 2020: (v1.10.2) - * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) - * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) - -* Feb 18, 2020: (v1.10.1) - * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) - * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) - * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) - -* Feb 4, 2020: (v1.10.0) - * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) - * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) - * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) - * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) - -
- -
- See changes prior to v1.10.0 - -* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). -* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) -* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. -* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. -* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) -* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. -* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) -* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features -* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) -* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) -* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. -* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) -* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) -* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) -* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. -* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. -* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) -* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. -* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) -* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. -* Nov 11, 2019: Reduce inflate memory use by 1KB. -* Nov 10, 2019: Less allocations in deflate bit writer. -* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. -* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) -* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) -* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) -* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) - -
- -
- See changes prior to v1.9.0 - -* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) -* Oct 3, 2019: Fix inconsistent results on broken zstd streams. -* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) -* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). -* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). -* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). -* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. -* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. -* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. -* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. -* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. -* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. -* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. -* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) -* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) -* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) -* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) -* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. -* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. -* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. -* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. -* June 17, 2019: zstd decompression bugfix. -* June 17, 2019: fix 32 bit builds. -* June 17, 2019: Easier use in modules (less dependencies). -* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. -* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. -* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. -* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! -* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. -* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. -* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). -* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. -* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). -* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. -* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. -* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. -* May 28, 2017: Reduce allocations when resetting decoder. -* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. -* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). -* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. -* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. -* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. -* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. -* Mar 24, 2016: Small speedup for level 1-3. -* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. -* Feb 19, 2016: Handle small payloads faster in level 1-3. -* Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. -* Feb 14, 2016: Snappy: Merge upstream changes. -* Feb 14, 2016: Snappy: Fix aggressive skipping. -* Feb 14, 2016: Snappy: Update benchmark. -* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. -* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. -* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. -* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. -* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. -* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. -* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. -* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. -* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! -* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). -* Nov 20 2015: Small optimization to bit writer on 64 bit systems. -* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). -* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. -* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file -* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. - -
- -# deflate usage - -The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: - -Typical speed is about 2x of the standard library packages. - -| old import | new import | Documentation | -|------------------|---------------------------------------|-------------------------------------------------------------------------| -| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | -| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | -| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | -| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | - -You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. - -The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). - -Currently there is only minor speedup on decompression (mostly CRC32 calculation). - -Memory usage is typically 1MB for a Writer. stdlib is in the same range. -If you expect to have a lot of concurrently allocated Writers consider using -the stateless compression described below. - -For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). - -To disable all assembly add `-tags=noasm`. This works across all packages. - -# Stateless compression - -This package offers stateless compression as a special option for gzip/deflate. -It will do compression but without maintaining any state between Write calls. - -This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. - -This is only relevant in cases where you expect to run many thousands of compressors concurrently, -but with very little activity. This is *not* intended for regular web servers serving individual requests. - -Because of this, the size of actual Write calls will affect output size. - -In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. - -For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) - -A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: - -```go - // replace 'ioutil.Discard' with your output. - gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) - if err != nil { - return err - } - defer gzw.Close() - - w := bufio.NewWriterSize(gzw, 4096) - defer w.Flush() - - // Write to 'w' -``` - -This will only use up to 4KB in memory when the writer is idle. - -Compression is almost always worse than the fastest compression level -and each write will allocate (a little) memory. - - -# Other packages - -Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): - -* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. -* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. -* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. -* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. -* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. -* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. -* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. - -# license - -This code is licensed under the same conditions as the original Go code. See LICENSE file. - - - - - +# compress + +This package provides various compression algorithms. + +* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. +* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. +* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). +* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. +* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. + +[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) +[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) +[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) + +# package usage + +Use `go get github.com/klauspost/compress@latest` to add it to your project. + +This package will support the current Go version and 2 versions back. + +* Use the `nounsafe` tag to disable all use of the "unsafe" package. +* Use the `noasm` tag to disable all assembly across packages. + +Use the links above for more information on each. + +# changelog + +* Jul 1st, 2026 [1.19.0](https://github.com/klauspost/compress/releases/tag/v1.19.0) + * zstd: Add true concurrent stream encodingin https://github.com/klauspost/compress/pull/1136 + * zstd: arm64 decoder asm by @lizthegrey in https://github.com/klauspost/compress/pull/1160 + * flate: Add inflate checkpoints in https://github.com/klauspost/compress/pull/1154 + * zstd: avoid unused BuildDict encoder allocation by @snissn in https://github.com/klauspost/compress/pull/1147 + * snappy/s2: Limit length of varint in `decodedLen` by @eustas in https://github.com/klauspost/compress/pull/1148 + * gzhttp: match qvalue parameter case-insensitively (RFC 7231) by @z9z in https://github.com/klauspost/compress/pull/1149 + * zip: add NameDecoder callback for legacy encoding rewrite by @SAY-5 in https://github.com/klauspost/compress/pull/1150 + * huff0: Allow building tables from histogram in https://github.com/klauspost/compress/pull/1155 + * huff0: Allow building table from oversized histogram in https://github.com/klauspost/compress/pull/1156 + * s2sx: Clean symlink targets in https://github.com/klauspost/compress/pull/1163 + +* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) + * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 + * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 + * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 + +* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) + * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). + +* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) + * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 + * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 + +* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED + * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 + * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 + * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 + * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 + * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 + * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 + * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 + * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 + +* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) + * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 + * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 + * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 + * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 + * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 + * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 + * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 + +
+ See changes to v1.17.x + +* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) + * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 + * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 + * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 + * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 + +* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) + * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 + * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 + * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 + * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 + * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 + +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + +* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) + * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 + * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 + +* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) + * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 + * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 + * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 + * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 + * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 +https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 + +* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) + * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 + * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 + * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 + * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 + * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 + +* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) + * fse: Fix max header size https://github.com/klauspost/compress/pull/881 + * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 + * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 + +* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) + * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 + +* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) + * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 + * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 + * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 + +* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) + * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 + * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 + * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 + * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 + * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 + * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 + +
+
+ See changes to v1.16.x + + +* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) + * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 + * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 + +* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) + * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 + * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 + * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 + * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 + +* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) + * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 + * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 + +* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) + * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 + * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 + * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 + * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 + * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 + * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + +* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) + * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 + * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 + * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 + * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 + * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 + +* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) + * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 + * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 + * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 + * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 + * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 + * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 +
+ +
+ See changes to v1.15.x + +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + +* Sept 16, 2022 (v1.15.10) + + * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 + * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 + * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 + * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 + * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 + * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 + * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 + +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 + +* July 13, 2022 (v1.15.8) + + * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 + * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 + * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 + * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 + * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 + * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 + * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 + +* June 29, 2022 (v1.15.7) + + * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 + * zip: Merge upstream https://github.com/klauspost/compress/pull/631 + * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 + * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 + * flate: Faster histograms https://github.com/klauspost/compress/pull/620 + * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 + +* June 3, 2022 (v1.15.6) + * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 + * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 + * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 + * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 + * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 + * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 + * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 + * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 + * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 + * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 + +* May 25, 2022 (v1.15.5) + * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 + * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 + * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 + * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 + * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 + * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 + * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 + * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 + * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 + + +* May 11, 2022 (v1.15.4) + * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) + * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) + * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) + * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) + +* May 5, 2022 (v1.15.3) + * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) + * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) + +* Apr 26, 2022 (v1.15.2) + * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) + * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) + * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) + * Minimum version is Go 1.16, added CI test on 1.18. + +* Mar 11, 2022 (v1.15.1) + * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) + * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) + * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) + * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) + * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) + +* Mar 3, 2022 (v1.15.0) + * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) + * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) + * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) + * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) + * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) + * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) + +Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. + +Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. + +While the release has been extensively tested, it is recommended to testing when upgrading. + +
+ +
+ See changes to v1.14.x + +* Feb 22, 2022 (v1.14.4) + * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) + * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) + * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 + * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) + +* Feb 17, 2022 (v1.14.3) + * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) + * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) + * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) + +* Jan 25, 2022 (v1.14.2) + * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) + * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) + * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) + * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) + * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) + * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) + +* Jan 11, 2022 (v1.14.1) + * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) + * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) + * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) + * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) + * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) +
+ +
+ See changes to v1.13.x + +* Aug 30, 2021 (v1.13.5) + * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) + * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) + * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) + * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) + +* Aug 12, 2021 (v1.13.4) + * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). + * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) + +* Aug 3, 2021 (v1.13.3) + * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) + * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) + * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) + * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) + * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) + * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) + +* Jun 14, 2021 (v1.13.1) + * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) + * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) + * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) + * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) + +* Jun 3, 2021 (v1.13.0) + * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. + * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) + * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) +
+ + +
+ See changes to v1.12.x + +* May 25, 2021 (v1.12.3) + * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) + * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) + * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) + +* Apr 27, 2021 (v1.12.2) + * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) + * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) + * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) + * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) + * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) + * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) + +* Apr 14, 2021 (v1.12.1) + * snappy package removed. Upstream added as dependency. + * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) + * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) + * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) + * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) + * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) + * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) +
+ +
+ See changes to v1.11.x + +* Mar 26, 2021 (v1.11.13) + * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) + * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) + * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) + * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) + +* Mar 5, 2021 (v1.11.12) + * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). + * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) + +* Mar 1, 2021 (v1.11.9) + * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) + * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) + * s2: Fix binaries. + +* Feb 25, 2021 (v1.11.8) + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. + * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) + * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) + * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) + * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) + +* Jan 14, 2021 (v1.11.7) + * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) + * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) + * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) + * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) + * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) + +* Jan 7, 2021 (v1.11.6) + * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) + * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) + +* Dec 20, 2020 (v1.11.4) + * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) + * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) + * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) + * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) + * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) + +* Nov 15, 2020 (v1.11.3) + * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) + * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) + +* Oct 11, 2020 (v1.11.2) + * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) + +* Oct 1, 2020 (v1.11.1) + * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) + +* Sept 8, 2020 (v1.11.0) + * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) + * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) + * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) +
+ +
+ See changes to v1.10.x + +* July 8, 2020 (v1.10.11) + * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) + * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) + +* June 23, 2020 (v1.10.10) + * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) + +* June 16, 2020 (v1.10.9): + * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) + * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) + * Fuzzit tests removed. The service has been purchased and is no longer available. + +* June 5, 2020 (v1.10.8): + * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) + +* June 1, 2020 (v1.10.7): + * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) + * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) + * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) + +* May 21, 2020: (v1.10.6) + * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) + * zstd: Stricter decompression checks. + +* April 12, 2020: (v1.10.5) + * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) + +* Apr 8, 2020: (v1.10.4) + * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) +* Mar 11, 2020: (v1.10.3) + * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) + * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) + * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) + * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) + * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) + +* Feb 27, 2020: (v1.10.2) + * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) + * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) + +* Feb 18, 2020: (v1.10.1) + * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) + * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) + * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) + +* Feb 4, 2020: (v1.10.0) + * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) + * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) + * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) + * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) + +
+ +
+ See changes prior to v1.10.0 + +* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). +* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) +* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. +* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. +* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) +* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. +* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) +* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features +* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) +* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) +* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. +* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) +* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) +* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) +* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. +* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. +* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) +* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. +* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) +* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. +* Nov 11, 2019: Reduce inflate memory use by 1KB. +* Nov 10, 2019: Less allocations in deflate bit writer. +* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. +* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) +* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) +* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) +* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) + +
+ +
+ See changes prior to v1.9.0 + +* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) +* Oct 3, 2019: Fix inconsistent results on broken zstd streams. +* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) +* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). +* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). +* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). +* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. +* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. +* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. +* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. +* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. +* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. +* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. +* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) +* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) +* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) +* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) +* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. +* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. +* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. +* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. +* June 17, 2019: zstd decompression bugfix. +* June 17, 2019: fix 32 bit builds. +* June 17, 2019: Easier use in modules (less dependencies). +* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. +* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. +* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. +* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! +* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. +* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. +* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). +* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. +* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). +* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. +* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. +* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. +* May 28, 2017: Reduce allocations when resetting decoder. +* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. +* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). +* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. +* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. +* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. +* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. +* Mar 24, 2016: Small speedup for level 1-3. +* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. +* Feb 19, 2016: Handle small payloads faster in level 1-3. +* Feb 19, 2016: Added faster level 2 + 3 compression modes. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. +* Feb 14, 2016: Snappy: Merge upstream changes. +* Feb 14, 2016: Snappy: Fix aggressive skipping. +* Feb 14, 2016: Snappy: Update benchmark. +* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. +* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. +* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. +* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. +* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. +* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. +* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. +* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. +* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! +* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). +* Nov 20 2015: Small optimization to bit writer on 64 bit systems. +* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). +* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. +* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file +* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. + +
+ +# deflate usage + +The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: + +Typical speed is about 2x of the standard library packages. + +| old import | new import | Documentation | +|------------------|---------------------------------------|-------------------------------------------------------------------------| +| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | +| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | +| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | +| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | + +You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. + +The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). + +Currently there is only minor speedup on decompression (mostly CRC32 calculation). + +Memory usage is typically 1MB for a Writer. stdlib is in the same range. +If you expect to have a lot of concurrently allocated Writers consider using +the stateless compression described below. + +For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). + +To disable all assembly add `-tags=noasm`. This works across all packages. + +# Stateless compression + +This package offers stateless compression as a special option for gzip/deflate. +It will do compression but without maintaining any state between Write calls. + +This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. + +This is only relevant in cases where you expect to run many thousands of compressors concurrently, +but with very little activity. This is *not* intended for regular web servers serving individual requests. + +Because of this, the size of actual Write calls will affect output size. + +In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. + +For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) + +A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: + +```go + // replace 'ioutil.Discard' with your output. + gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) + if err != nil { + return err + } + defer gzw.Close() + + w := bufio.NewWriterSize(gzw, 4096) + defer w.Flush() + + // Write to 'w' +``` + +This will only use up to 4KB in memory when the writer is idle. + +Compression is almost always worse than the fastest compression level +and each write will allocate (a little) memory. + + +# Other packages + +Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): + +* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. +* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. +* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. +* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. +* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. +* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. +* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. + +# license + +This code is licensed under the same conditions as the original Go code. See LICENSE file. + + + + + diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md index ea7324da6..27d8ed56f 100644 --- a/vendor/github.com/klauspost/compress/fse/README.md +++ b/vendor/github.com/klauspost/compress/fse/README.md @@ -1,79 +1,79 @@ -# Finite State Entropy - -This package provides Finite State Entropy encoding and decoding. - -Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) -encoding provides a fast near-optimal symbol encoding/decoding -for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) - -## News - - * Feb 2018: First implementation released. Consider this beta software for now. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `(error)` | An internal error occurred. | - -As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). - -# Performance - -A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. -All compression functions are currently only running on the calling goroutine so only one core will be used per block. - -The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input -is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be -beneficial to transpose all your input values down by 64. - -With moderate block sizes around 64k speed are typically 200MB/s per core for compression and -around 300MB/s decompression speed. - -The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. - -# Plans - -At one point, more internals will be exposed to facilitate more "expert" usage of the components. - -A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +# Finite State Entropy + +This package provides Finite State Entropy encoding and decoding. + +Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) +encoding provides a fast near-optimal symbol encoding/decoding +for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) + +## News + + * Feb 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `(error)` | An internal error occurred. | + +As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). + +# Performance + +A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. +All compression functions are currently only running on the calling goroutine so only one core will be used per block. + +The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input +is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be +beneficial to transpose all your input values down by 64. + +With moderate block sizes around 64k speed are typically 200MB/s per core for compression and +around 300MB/s decompression speed. + +The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. + +# Plans + +At one point, more internals will be exposed to facilitate more "expert" usage of the components. + +A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md index 8b6e5c663..26d5101b3 100644 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -1,89 +1,89 @@ -# Huff0 entropy compression - -This package provides Huff0 encoding and decoding as used in zstd. - -[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), -a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU -(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) - -## News - -This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. - -This ensures that most functionality is well tested. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and -[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | -| `(error)` | An internal error occurred. | - - -As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. - -## Tables and re-use - -Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. - -The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) -that controls this behaviour. See the documentation for details. This can be altered between each block. - -Do however note that this information is *not* stored in the output block and it is up to the users of the package to -record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, -based on the boolean reported back from the CompressXX call. - -If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the -[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. - -## Decompressing - -The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). -This will initialize the decoding tables. -You can supply the complete block to `ReadTable` and it will return the data part of the block -which can be given to the decompressor. - -Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) -or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. - -For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. - -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking -changes will likely not be accepted. If in doubt open an issue before writing the PR. +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +## News + +This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. + +This ensures that most functionality is well tested. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. diff --git a/vendor/github.com/klauspost/compress/huff0/build_table.go b/vendor/github.com/klauspost/compress/huff0/build_table.go new file mode 100644 index 000000000..e3757c87e --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/build_table.go @@ -0,0 +1,168 @@ +package huff0 + +import "errors" + +// BuildCTable builds a Huffman compression table from a precomputed symbol +// histogram and installs it as the previous (reuse) table on s. +// +// After this call: +// - EstimateSize/CanUseTable can probe the table against other histograms. +// - Compress1X/Compress4X with Reuse = ReusePolicyMust will encode without +// emitting a new table header. +// - TransferCTable can hand the table to a sibling Scratch. +// +// count[i] is the number of occurrences of symbol i. The histogram must have +// at least 2 distinct non-zero symbols; ErrUseRLE is returned for a single +// symbol and an error is returned for an empty histogram. +func (s *Scratch) BuildCTable(count *[256]uint32) error { + if s == nil { + return errors.New("huff0: BuildCTable on nil Scratch") + } + if count == nil { + return errors.New("huff0: nil count passed to BuildCTable") + } + var err error + s, err = s.prepare(nil) + if err != nil { + return err + } + s.count = *count + var total, maxCount int + var symLen uint16 + for i, v := range s.count { + total += int(v) + if int(v) > maxCount { + maxCount = int(v) + } + if v != 0 { + symLen = uint16(i) + 1 + } + } + if total == 0 { + return errors.New("huff0: empty histogram") + } + if symLen < 2 || maxCount == total { + return ErrUseRLE + } + // huff0's internal rank table assumes total ≤ BlockSizeMax (it uses + // highBit32(count+1) + 1 as a rank index into a fixed-size array). + // Histograms summed across multiple blocks can exceed that; scale the + // counts down preserving the distribution. Non-zero entries round up so + // rare symbols stay representable. + if total > BlockSizeMax { + shift := uint(0) + for total>>shift > BlockSizeMax { + shift++ + } + round := uint32(1<> shift + if scaled == 0 { + scaled = 1 + } + s.count[i] = scaled + newTotal += int(scaled) + if int(scaled) > newMax { + newMax = int(scaled) + } + } + total = newTotal + maxCount = newMax + if maxCount == total { + return ErrUseRLE + } + } + s.symbolLen = symLen + s.maxCount = maxCount + s.srcLen = total + if err := s.buildCTable(); err != nil { + return err + } + if cap(s.prevTable) < len(s.cTable) { + s.prevTable = make(cTable, 0, maxSymbolValue+1) + } + s.prevTable = s.prevTable[:len(s.cTable)] + copy(s.prevTable, s.cTable) + s.prevTableLog = s.actualTableLog + // Force the next Compress* to recount from real input. + s.clearCount = true + s.maxCount = 0 + return nil +} + +// EstimateSize returns an estimated compressed payload size in bytes for the +// supplied histogram using the table currently stored in prevTable. It returns +// -1 when the table cannot encode every non-zero symbol of hist (i.e. when +// CanUseTable would return false). The estimate excludes the table header. +func (s *Scratch) EstimateSize(hist *[256]uint32) int { + if s == nil || hist == nil || len(s.prevTable) == 0 { + return -1 + } + pt := s.prevTable + nbBits := uint32(7) + for i, v := range hist { + if v == 0 { + continue + } + if i >= len(pt) || pt[i].nBits == 0 { + return -1 + } + nbBits += uint32(pt[i].nBits) * v + } + return int(nbBits >> 3) +} + +// CanUseTable reports whether the table in prevTable can encode every +// non-zero symbol present in hist. +func (s *Scratch) CanUseTable(hist *[256]uint32) bool { + if s == nil || hist == nil || len(s.prevTable) == 0 { + return false + } + pt := s.prevTable + for i, v := range hist { + if v == 0 { + continue + } + if i >= len(pt) || pt[i].nBits == 0 { + return false + } + } + return true +} + +// AppendTable serializes the table currently stored in prevTable (e.g. as +// installed by BuildCTable or carried over from a previous Compress call) +// into a self-delimiting zstd-style header and appends it to dst. The +// returned slice can be parsed back by ReadTable. +func (s *Scratch) AppendTable(dst []byte) ([]byte, error) { + if s == nil || len(s.prevTable) == 0 { + return dst, errors.New("huff0: AppendTable with empty table") + } + // cTable.write reads s.actualTableLog, s.symbolLen, s.huffWeight, s.fse + // and writes into s.Out. Save/restore Out so we don't disturb in-flight + // compression buffers. + saveOut := s.Out + saveTL := s.actualTableLog + saveSL := s.symbolLen + if s.fse == nil { + // Lazily init in case AppendTable is called on a fresh Scratch. + if _, err := s.prepare(nil); err != nil { + return dst, err + } + saveOut = s.Out + } + s.Out = s.Out[:0] + s.actualTableLog = s.prevTableLog + s.symbolLen = uint16(len(s.prevTable)) + if err := s.prevTable.write(s); err != nil { + s.Out, s.actualTableLog, s.symbolLen = saveOut, saveTL, saveSL + return dst, err + } + dst = append(dst, s.Out...) + s.Out, s.actualTableLog, s.symbolLen = saveOut, saveTL, saveSL + return dst, nil +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 2d6ef64be..7035d656d 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,222 +1,49 @@ //go:build amd64 && !appengine && !noasm && gc -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. +// amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 import ( - "errors" - "fmt" - "github.com/klauspost/compress/internal/cpuinfo" ) -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress4x_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog > 8. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) -// decompress4x_8b_loop_x86 is an x86 assembler implementation +// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog <= 8 which decodes 4 entries // per loop. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 - -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := range 3 { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := min(offset+remainBytes, len(out)) - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress1x_main_loop_amd64 is an x86 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation +// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_amd64(ctx) } -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_amd64(ctx) +} - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(ctx) + } else { + decompress1x_main_loop_amd64(ctx) } - return dst, br.close() } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index c4c7ab2d1..c5d4a710a 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build amd64 && !appengine && !noasm && gc +//go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_amd64(SB), $0-8 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go new file mode 100644 index 000000000..8ba3c810e --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -0,0 +1,37 @@ +//go:build arm64 && !appengine && !noasm && gc + +// arm64 stubs and dispatch for the asm loops used by decompress_asm.go. +// The asm (decompress_arm64.s) is generated by the avo arm64 lowering +// printer from the same source as the amd64 asm; see _generate/gen.go. +package huff0 + +// decompress4x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog > 8. +// +//go:noescape +func decompress4x_main_loop_arm64(ctx *decompress4xContext) + +// decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog <= 8 which decodes 4 entries +// per loop. +// +//go:noescape +func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext) + +// decompress1x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog > 8. +// +//go:noescape +func decompress1x_main_loop_arm64(ctx *decompress1xContext) + +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_arm64(ctx) +} + +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_arm64(ctx) +} + +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + decompress1x_main_loop_arm64(ctx) +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s new file mode 100644 index 000000000..e16012721 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -0,0 +1,851 @@ +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && !appengine && !noasm && gc + +// func decompress4x_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R0 + MOVBU 8(R0), R6 + MOVD 16(R0), R3 + MOVD 48(R0), R5 + MOVD 24(R0), R7 + MOVD 32(R0), R8 + MOVD (R0), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD (R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 24(R9) + ORR R12, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVH R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 48(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 72(R9) + ORR R12, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 96(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 120(R9) + ORR R12, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 144(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 168(R9) + ORR R12, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x02, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R1 + MOVBU 8(R1), R6 + MOVD 16(R1), R3 + MOVD 48(R1), R5 + MOVD 24(R1), R7 + MOVD 32(R1), R8 + MOVD (R1), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD (R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 24(R9) + ORR R13, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVW R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 48(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 72(R9) + ORR R13, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 96(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 120(R9) + ORR R13, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 144(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 168(R9) + ORR R13, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x04, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +TEXT ·decompress1x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R1 + MOVD 16(R1), R2 + MOVD 24(R1), R3 + CMP $0x04, R3 + BLO error_max_decoded_size_exceeded + ADD R3, R2, R3 + MOVD (R1), R5 + MOVD (R5), R7 + MOVD 24(R5), R8 + MOVD 32(R5), R9 + MOVBU 40(R5), R10 + MOVD 32(R1), R5 + MOVBU 8(R1), R6 + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + ADD $4, R2, R1 + CMP R3, R1 + BGE error_max_decoded_size_exceeded + + // Decode 4 values + CMP $0x20, R10 + BLT bitReader_fillFast_1_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_1_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + CMP $0x20, R10 + BLT bitReader_fillFast_2_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_2_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + + // Store the decoded values + MOVW R0, (R2) + ADD $0x04, R2, R2 + +loop_condition: + CMP $0x08, R8 + BGE main_loop + + // Update ctx structure + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R2, R2 + MOVD R2, 40(R0) + MOVD (R0), R0 + MOVD R8, 24(R0) + MOVD R9, 32(R0) + MOVB R10, 40(R0) + RET + + // Report error +error_max_decoded_size_exceeded: + MOVD ctx+0(FP), R0 + MOVD $-1, R1 + MOVD R1, 40(R0) + RET + +// skipped decompress1x_main_loop_bmi2 (generic twin preferred on arm64) diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go new file mode 100644 index 000000000..4854dd436 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -0,0 +1,193 @@ +//go:build (amd64 || arm64) && !appengine && !noasm && gc + +// This file contains the specialisation of Decoder.Decompress4X +// and Decoder.Decompress1X that use an asm implementation of their main loops. +// The asm function stubs and any per-arch dispatch live in decompress_amd64.go +// and decompress_arm64.go. +package huff0 + +import ( + "errors" + "fmt" +) + +// fallback8BitSize is the size where using Go version is faster. +const fallback8BitSize = 800 + +type decompress4xContext struct { + pbr *[4]bitReaderShifted + peekBits uint8 + out *byte + dstEvery int + tbl *dEntrySingle + decoded int + limit *byte +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(src) < 6+(4*1) { + return nil, errors.New("input too small") + } + + use8BitTables := d.actualTableLog <= 8 + if cap(dst) < fallback8BitSize && use8BitTables { + return d.decompress4X8bit(dst, src) + } + + var br [4]bitReaderShifted + // Decode "jump table" + start := 6 + for i := range 3 { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + var decoded int + + if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + ctx := decompress4xContext{ + pbr: &br, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + out: &out[0], + dstEvery: dstEvery, + tbl: &single[0], + limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + } + if use8BitTables { + decompress4x_8b_main_loop_asm(&ctx) + } else { + decompress4x_main_loop_asm(&ctx) + } + + decoded = ctx.decoded + out = out[decoded/4:] + } + + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) + for i := range br { + offset := dstEvery * i + endsAt := min(offset+remainBytes, len(out)) + br := &br[i] + bitsLeft := br.remaining() + for bitsLeft > 0 { + br.fill() + if offset >= endsAt { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + if offset != endsAt { + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +type decompress1xContext struct { + pbr *bitReaderShifted + peekBits uint8 + out *byte + outCap int + tbl *dEntrySingle + decoded int +} + +// Error reported by asm implementations +const error_max_decoded_size_exeeded = -1 + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:maxDecodedSize] + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + + if maxDecodedSize >= 4 { + ctx := decompress1xContext{ + pbr: &br, + out: &dst[0], + outCap: maxDecodedSize, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + tbl: &d.dt.single[0], + } + + decompress1x_main_loop_asm(&ctx) + if ctx.decoded == error_max_decoded_size_exeeded { + return nil, ErrMaxDecodedSizeExceeded + } + + dst = dst[:ctx.decoded] + } + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 610392322..e1bca5c68 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/internal/snapref/decode.go b/vendor/github.com/klauspost/compress/internal/snapref/decode.go index a2c82fcd2..584b7574b 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/decode.go @@ -31,7 +31,7 @@ func DecodedLen(src []byte) (int, error) { // that the length header occupied. func decodedLen(src []byte) (blockLen, headerLen int, err error) { v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { + if n <= 0 || n > 5 || v > 0xffffffff { return 0, 0, ErrCorrupt } diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index c11d7fa28..a5aeeaed0 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -75,14 +75,47 @@ The above is fine for big encodes. However, whenever possible try to *reuse* the To reuse the encoder, you can use the `Reset(io.Writer)` function to change to another output. This will allow the encoder to reuse all resources and avoid wasteful allocations. -Currently stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part -of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change +By default, stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part +of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change in the future. So if you want to limit concurrency for future updates, specify the concurrency you would like. If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)` which will compress input as each block is completed, blocking on writes until each has completed. +#### Parallel Stream Compression + +For maximum throughput on large streams, use `WithConcurrentBlocks(true)` together with +`WithEncoderConcurrency(n)` where n is the number of CPU cores you want to use. +This splits the input into large sections (jobs) that are compressed simultaneously by multiple goroutines, +similar to how the C zstd library does multithreaded compression. + +```Go +enc, err := zstd.NewWriter(out, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderConcurrency(runtime.GOMAXPROCS(0)), + zstd.WithConcurrentBlocks(true), +) +``` + +Each non-first job receives an overlap prefix from the previous job for match context, +so compression ratio is only marginally affected. Output is flushed in order, +producing a valid single-frame zstd stream. + +Benchmark on 1.8GB GOB stream (AMD Ryzen 9 9950X): + +| Level | 1 thread | 4 threads | 16 threads | 1T ratio | 16T ratio | +|---------|:----------:|:------------------:|:-------------------:|:--------:|:---------:| +| fastest | 783 MB/s | 2950 MB/s (3.8×) | 6939 MB/s (8.9×) | 12.24% | 12.26% | +| default | 728 MB/s | 2533 MB/s (3.5×) | 5340 MB/s (7.3×) | 10.67% | 10.68% | +| better | 434 MB/s | 1105 MB/s (2.5×) | 2206 MB/s (5.1×) | 9.14% | 9.21% | +| best | 129 MB/s | 367 MB/s (2.8×) | 884 MB/s (6.8×) | 8.48% | 8.63% | + +Notes: +* Not compatible with dictionary encoding. +* `Flush()` dispatches the current partial job, so latency-sensitive callers can force output. +* `EncodeAll` is unaffected — it uses its own concurrency via the encoder pool. + You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined compression settings can be specified. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 2329e996f..51f9da038 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -400,8 +400,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } } var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize + // Decoder.Decompress* uses cap(dst) for the size limit. Do not write + // MaxDecodedSize on hist.huffTree: with a trained dictionary that + // pointer aliases the shared dict.litEnc and concurrent DecodeAll races. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -673,10 +674,6 @@ func (b *blockDec) executeSequences(hist *history) error { hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } } hist.decoders.windowSize = hist.windowSize hist.decoders.out = b.dst[:0] diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 2ffbfdf37..67f40e791 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -230,7 +230,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { } block := blockEnc{lowMem: false} block.init() - enc := encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}}) + var enc encoder if o.Level != 0 { eOpts := encoderOptions{ level: o.Level, @@ -242,6 +242,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { enc = eOpts.encoder() } else { o.Level = SpeedBestCompression + enc = encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}}) } var ( remain [256]int @@ -295,40 +296,81 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset > 3 { newOffsets[offset-3]++ } else { - newOffsets[uint32(o.Offsets[offset-1])]++ + // Repeat codes reference the training Offsets. Skip unset + // (zero) entries so they are not ranked as real offsets. + prev := o.Offsets[offset-1] + if prev > 0 { + newOffsets[uint32(prev)]++ + } } } } // Find most used offsets. var sortedOffsets []uint32 for k := range newOffsets { + if k == 0 { + continue + } sortedOffsets = append(sortedOffsets, k) } sort.Slice(sortedOffsets, func(i, j int) bool { a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { + ca, cb := newOffsets[a], newOffsets[b] + if ca == cb { // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] + return a > b } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] + return ca > cb }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) + if debug { + print("Offsets:") + for i, v := range sortedOffsets { + if i > 20 { + break } - println("") + printf("[%d: %d],", v, newOffsets[v]) + } + println("") + } + // Dictionary recent-offsets must be three positive values within the + // history. Ranked matches may be fewer (or empty when only unset + // repeat codes were seen), so fill remaining slots with defaults. + used := make(map[int]bool, 3) + var finalOffsets [3]int + nOff := 0 + for _, v := range sortedOffsets { + iv := int(v) + if iv <= 0 || iv > len(hist) || used[iv] { + continue + } + finalOffsets[nOff] = iv + used[iv] = true + nOff++ + if nOff == 3 { + break } - - sortedOffsets = sortedOffsets[:3] } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) + for _, def := range []int{1, 4, 8} { + if nOff == 3 { + break + } + if def <= len(hist) && !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + for def := 1; nOff < 3 && def <= len(hist); def++ { + if !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + if nOff < 3 { + return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist)) } + o.Offsets = finalOffsets if debug { println("New repeat offsets", o.Offsets) } @@ -336,6 +378,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if nUsed == 0 || seqs == 0 { return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) } + if litTotal == 0 { + return nil, errors.New("0 literals found") + } if debug { println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) } @@ -516,11 +561,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) out.Write(hist) + if _, err := loadDict(out.Bytes()); err != nil { + return nil, fmt.Errorf("built dictionary failed validation: %w", err) + } if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } i, err := InspectDictionary(out.Bytes()) if err != nil { panic(err) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index c4de134a7..c4fea575d 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -128,6 +128,34 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { return int32(matchLen(src[s:], src[t:])) } +// resetBasePrefix resets the encoder state and loads prefix as initial history. +// This is used for parallel job encoding where non-first jobs need overlap context. +// Rep offsets are set to defaults [1,4,8] (invalidated, matching C behavior). +func (e *fastBase) resetBasePrefix(prefix []byte) { + if e.blk == nil { + e.blk = &blockEnc{lowMem: e.lowMem} + e.blk.init() + } else { + e.blk.reset(nil) + } + e.blk.initNewEncode() + if e.crc == nil { + e.crc = xxhash.New() + } else { + e.crc.Reset() + } + e.blk.dictLitEnc = nil + e.ensureHist(len(prefix) + maxCompressedBlockSize) + // Bump cur so old table entries fall outside the window. + // When cur >= bufferReset, leave it; the first Encode call + // will shift/clear tables, preserving valid prefix entries. + if e.cur < e.bufferReset { + e.cur += e.maxMatchOff + int32(len(e.hist)) + } + e.hist = e.hist[:0] + e.hist = append(e.hist, prefix...) +} + // Reset the encoding table. func (e *fastBase) resetBase(d *dict, singleBlock bool) { if e.blk == nil { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index 851799322..c71382dde 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -551,3 +551,18 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { // Reset table to initial state copy(e.table[:], e.dictTable) } + +func (e *bestFastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur; i < end; i++ { + cv := load6432(prefix, i-e.cur) + h := hashLen(cv, bestLongTableBits, bestLongLen) + e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset} + h0 := hashLen(cv, bestShortTableBits, bestShortLen) + e.table[h0] = prevEntry{offset: i, prev: e.table[h0].offset} + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 3305f0924..523d57f3a 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -1096,6 +1096,20 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { } } +func (e *betterFastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur; i < end; i += 2 { + cv := load6432(prefix, i-e.cur) + h := hashLen(cv, betterLongTableBits, betterLongLen) + e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset} + e.table[hashLen(cv>>8, betterShortTableBits, betterShortLen)] = tableEntry{val: uint32(cv >> 8), offset: i + 1} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) @@ -1229,6 +1243,10 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { e.allDirty = false } +func (e *betterFastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) { e.longTableShardDirty[entryNum/betterLongTableShardSize] = true } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 2fb6da112..712ba7ab5 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -1037,6 +1037,18 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { } } +func (e *doubleFastEncoder) ResetPrefix(prefix []byte) { + e.fastEncoder.ResetPrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur + 1; i < end; i += 2 { + cv := load6432(prefix, i-e.cur) + e.longTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{val: uint32(cv), offset: i} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { allDirty := e.allDirty @@ -1102,6 +1114,10 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } } +func (e *doubleFastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) { e.longTableShardDirty[entryNum/dLongTableShardSize] = true } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 5e104f1a4..06045e246 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -797,6 +797,19 @@ func (e *fastEncoder) Reset(d *dict, singleBlock bool) { } } +func (e *fastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + // Index every 4th + for i := e.cur + 1; i < end; i += 4 { + cv := load6432(prefix, i-e.cur) + e.table[hashLen(cv, tableBits, tableFastHashLen)] = tableEntry{val: uint32(cv), offset: i} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) @@ -866,6 +879,10 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { e.allDirty = false } +func (e *fastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *fastEncoderDict) markAllShardsDirty() { e.allDirty = true } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_jobs.go b/vendor/github.com/klauspost/compress/zstd/enc_jobs.go new file mode 100644 index 000000000..95ce67ac0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_jobs.go @@ -0,0 +1,352 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "fmt" + rdebug "runtime/debug" + "sync" +) + +type encJob struct { + prefix []byte // overlap from previous job (nil for first) + input []byte // job's own input data (swapped from filling) + last bool // last block of last job gets last=true + output []byte // compressed blocks (filled by worker) + err error // encoding error + done chan struct{} // closed when complete +} + +type jobState struct { + jobSize int + overlapSize int + filling []byte // accumulates input up to jobSize + nextPrefix []byte // overlap prefix prepared for the next dispatched job + + jobSeq int // next job sequence number + + jobCh chan *encJob // dispatch to workers + resultCh chan *encJob // ordered results to flusher + + workerWg sync.WaitGroup + flusherWg sync.WaitGroup + + mu sync.Mutex + flushedSeq int // last flushed sequence number + cond *sync.Cond + + flusherErr error + started bool + + inputPool sync.Pool // *[]byte buffers of jobSize cap + outputPool sync.Pool // *[]byte buffers for compressed output + overlapPool sync.Pool // *[]byte buffers for overlap prefixes +} + +func (e *Encoder) startJobWorkers() { + js := &e.state.jobs + n := e.o.concurrent + js.jobCh = make(chan *encJob, n) + js.resultCh = make(chan *encJob, n) + js.flushedSeq = 0 + js.cond = sync.NewCond(&js.mu) + + // Workers borrow encoders from the shared e.encoders pool per-job. + // Ensure the pool is initialized before any worker tries to borrow. + e.init.Do(e.initialize) + + for range n { + js.workerWg.Add(1) + go e.jobWorker() + } + js.flusherWg.Add(1) + go e.jobFlusher() + js.started = true +} + +func (e *Encoder) jobWorker() { + js := &e.state.jobs + defer js.workerWg.Done() + for job := range js.jobCh { + enc := <-e.encoders + e.compressJob(enc, job) + e.encoders <- enc + close(job.done) + } +} + +func (e *Encoder) compressJob(enc encoder, job *encJob) { + defer func() { + if r := recover(); r != nil { + job.err = fmt.Errorf("panic in parallel job: %v", r) + rdebug.PrintStack() + } + }() + + if len(job.prefix) > 0 { + enc.ResetPrefix(job.prefix) + } else { + enc.Reset(nil, false) + } + + data := job.input + if len(data) == 0 && job.last { + blk := enc.Block() + blk.reset(nil) + blk.last = true + blk.encodeRaw(nil) + job.output = append(job.output, blk.output...) + return + } + + blk := enc.Block() + for len(data) > 0 { + todo := data + if len(todo) > e.o.blockSize { + todo = todo[:e.o.blockSize] + } + data = data[len(todo):] + + blk.pushOffsets() + enc.Encode(blk, todo) + blk.last = len(data) == 0 && job.last + + err := blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy) + if err != nil { + job.err = err + return + } + job.output = append(job.output, blk.output...) + blk.reset(nil) + } +} + +func (js *jobState) getInputBuf(size int) []byte { + if v := js.inputPool.Get(); v != nil { + bp := v.(*[]byte) + b := *bp + if cap(b) >= size { + return b[:0] + } + } + return make([]byte, 0, size) +} + +func (js *jobState) putInputBuf(b []byte) { + if cap(b) > 0 { + b = b[:0] + js.inputPool.Put(&b) + } +} + +func (js *jobState) getOutputBuf(size int) []byte { + if v := js.outputPool.Get(); v != nil { + bp := v.(*[]byte) + b := *bp + if cap(b) >= size { + return b[:0] + } + } + return make([]byte, 0, size) +} + +func (js *jobState) putOutputBuf(b []byte) { + if cap(b) > 0 { + b = b[:0] + js.outputPool.Put(&b) + } +} + +func (js *jobState) getOverlapBuf(size int) []byte { + if v := js.overlapPool.Get(); v != nil { + bp := v.(*[]byte) + b := *bp + if cap(b) >= size { + return b[:size] + } + } + return make([]byte, size) +} + +func (js *jobState) putOverlapBuf(b []byte) { + if cap(b) > 0 { + b = b[:0] + js.overlapPool.Put(&b) + } +} + +func (e *Encoder) jobFlusher() { + js := &e.state.jobs + defer js.flusherWg.Done() + for job := range js.resultCh { + <-job.done + // Worker has fully exited compressJob, so the prefix is no longer + // in use. Return it to the pool regardless of outcome. + if job.prefix != nil { + js.putOverlapBuf(job.prefix) + job.prefix = nil + } + if job.err != nil { + js.mu.Lock() + js.flusherErr = job.err + js.cond.Broadcast() + js.mu.Unlock() + for range js.resultCh { + } + return + } + if len(job.output) > 0 { + _, err := e.state.w.Write(job.output) + if err != nil { + js.mu.Lock() + js.flusherErr = err + js.cond.Broadcast() + js.mu.Unlock() + for range js.resultCh { + } + return + } + e.state.nWritten += int64(len(job.output)) + } + // Return buffers to pools. + js.putInputBuf(job.input) + js.putOutputBuf(job.output) + job.input = nil + job.output = nil + + js.mu.Lock() + js.flushedSeq++ + js.cond.Broadcast() + js.mu.Unlock() + } +} + +func (e *Encoder) shutdownJobWorkers() { + js := &e.state.jobs + if !js.started { + return + } + close(js.jobCh) + js.workerWg.Wait() + close(js.resultCh) + js.flusherWg.Wait() + js.started = false +} + +// waitAllJobs blocks until all dispatched jobs have been flushed. +func (e *Encoder) waitAllJobs() { + js := &e.state.jobs + if !js.started { + return + } + js.mu.Lock() + for js.flushedSeq < js.jobSeq && js.flusherErr == nil { + js.cond.Wait() + } + js.mu.Unlock() +} + +func (e *Encoder) dispatchJob(final bool) error { + s := &e.state + js := &s.jobs + + js.mu.Lock() + fErr := js.flusherErr + js.mu.Unlock() + if fErr != nil { + return fErr + } + + if !s.headerWritten { + // Single-block optimization: fall through to encodeAll path. + if final && len(js.filling) > 0 && len(js.filling) <= e.o.blockSize { + s.current = e.encodeAll(s.encoder, js.filling, s.current[:0]) + var n2 int + n2, s.err = s.w.Write(s.current) + if s.err != nil { + return s.err + } + s.nWritten += int64(n2) + s.nInput += int64(len(js.filling)) + s.current = s.current[:0] + js.filling = js.filling[:0] + s.headerWritten = true + s.fullFrameWritten = true + s.eofWritten = true + return nil + } + if final && len(js.filling) == 0 && !e.o.fullZero { + s.headerWritten = true + s.fullFrameWritten = true + s.eofWritten = true + return nil + } + + var tmp [maxHeaderSize]byte + fh := frameHeader{ + ContentSize: uint64(s.frameContentSize), + WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)), + SingleSegment: false, + Checksum: e.o.crc, + DictID: 0, + } + dst := fh.appendTo(tmp[:0]) + var n2 int + n2, s.err = s.w.Write(dst) + if s.err != nil { + return s.err + } + s.nWritten += int64(n2) + s.headerWritten = true + } + + if len(js.filling) == 0 && !final { + return nil + } + + if !js.started { + e.startJobWorkers() + } + + // Estimate output size for pooled buffer. + outputEst := max(len(js.filling)/2, 512) + + job := &encJob{ + last: final, + done: make(chan struct{}), + output: js.getOutputBuf(outputEst), + } + + // Each job owns its prefix slice; the flusher returns it to the pool + // after <-job.done, so workers and dispatch never share a buffer. + if js.nextPrefix != nil { + job.prefix = js.nextPrefix + js.nextPrefix = nil + } + + // Build the next job's prefix from the tail of this job's input. + if !final && len(js.filling) > 0 { + overlapLen := min(js.overlapSize, len(js.filling)) + np := js.getOverlapBuf(overlapLen) + copy(np, js.filling[len(js.filling)-overlapLen:]) + js.nextPrefix = np + } + + // Swap filling buffer into job — zero-copy for the input data. + job.input = js.filling + js.filling = js.getInputBuf(js.jobSize) + + s.nInput += int64(len(job.input)) + js.jobSeq++ + + if final { + s.eofWritten = true + } + + js.resultCh <- job + js.jobCh <- job + + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 0f2a00a00..6ee96d873 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -38,6 +38,7 @@ type encoder interface { WindowSize(size int64) int32 UseBlock(*blockEnc) Reset(d *dict, singleBlock bool) + ResetPrefix(prefix []byte) } type encoderState struct { @@ -60,6 +61,9 @@ type encoderState struct { wg sync.WaitGroup // This waitgroup indicates we have a block encoding/writing. wWg sync.WaitGroup + + // Parallel job state (used when concurrentBlocks is enabled). + jobs jobState } // NewWriter will create a new Zstandard encoder. @@ -74,6 +78,9 @@ func NewWriter(w io.Writer, opts ...EOption) (*Encoder, error) { return nil, err } } + if e.o.concurrentBlocks && (e.o.dict != nil || e.o.concurrent <= 1) { + e.o.concurrentBlocks = false + } if w != nil { e.Reset(w) } @@ -95,12 +102,31 @@ func (e *Encoder) initialize() { // as a new, independent stream. func (e *Encoder) Reset(w io.Writer) { s := &e.state + + if e.o.concurrentBlocks { + e.shutdownJobWorkers() + js := &s.jobs + js.jobSize = e.o.jobSize() + js.overlapSize = e.o.overlapSize() + // js.filling is allocated lazily on first Write/ReadFrom so callers + // that only use EncodeAll don't pay the (up to ~32 MB) jobSize cost. + js.filling = js.filling[:0] + if js.nextPrefix != nil { + js.putOverlapBuf(js.nextPrefix) + js.nextPrefix = nil + } + js.jobSeq = 0 + js.flushedSeq = 0 + js.flusherErr = nil + js.started = false + } + s.wg.Wait() s.wWg.Wait() if cap(s.filling) == 0 { s.filling = make([]byte, 0, e.o.blockSize) } - if e.o.concurrent > 1 { + if e.o.concurrent > 1 && !e.o.concurrentBlocks { if cap(s.current) == 0 { s.current = make([]byte, 0, e.o.blockSize) } @@ -145,6 +171,9 @@ func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error { } } hasDict := e.o.dict != nil + if e.o.concurrentBlocks && hasDict { + e.o.concurrentBlocks = false + } if hadDict != hasDict { // Dict presence changed — encoder type must be recreated. e.state.encoder = nil @@ -176,6 +205,49 @@ func (e *Encoder) Write(p []byte) (n int, err error) { if s.eofWritten { return 0, ErrEncoderClosed } + if e.o.concurrentBlocks { + return e.writeJobs(p) + } + return e.writeBlocks(p) +} + +func (e *Encoder) writeJobs(p []byte) (n int, err error) { + s := &e.state + js := &s.jobs + jobSize := js.jobSize + if cap(js.filling) == 0 && len(p) > 0 { + js.filling = make([]byte, 0, jobSize) + } + for len(p) > 0 { + if len(p)+len(js.filling) < jobSize { + if e.o.crc { + _, _ = s.encoder.CRC().Write(p) + } + js.filling = append(js.filling, p...) + return n + len(p), nil + } + add := p + if len(p)+len(js.filling) > jobSize { + add = add[:jobSize-len(js.filling)] + } + if e.o.crc { + _, _ = s.encoder.CRC().Write(add) + } + js.filling = append(js.filling, add...) + p = p[len(add):] + n += len(add) + if len(js.filling) < jobSize { + return n, nil + } + if err := e.dispatchJob(false); err != nil { + return n, err + } + } + return n, nil +} + +func (e *Encoder) writeBlocks(p []byte) (n int, err error) { + s := &e.state for len(p) > 0 { if len(p)+len(s.filling) < e.o.blockSize { if e.o.crc { @@ -374,6 +446,10 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { println("Using ReadFrom") } + if e.o.concurrentBlocks { + return e.readFromJobs(r) + } + // Flush any current writes. if len(e.state.filling) > 0 { if err := e.nextBlock(false); err != nil { @@ -387,7 +463,6 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { if e.o.crc { _, _ = e.state.encoder.CRC().Write(src[:n2]) } - // src is now the unfilled part... src = src[n2:] n += int64(n2) switch err { @@ -420,15 +495,63 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { } } +func (e *Encoder) readFromJobs(r io.Reader) (n int64, err error) { + js := &e.state.jobs + jobSize := js.jobSize + + // Flush any current filling. + if len(js.filling) > 0 { + if err := e.dispatchJob(false); err != nil { + return 0, err + } + } + + if cap(js.filling) < jobSize { + js.filling = make([]byte, 0, jobSize) + } + js.filling = js.filling[:jobSize] + src := js.filling + for { + n2, err := r.Read(src) + if e.o.crc { + _, _ = e.state.encoder.CRC().Write(src[:n2]) + } + src = src[n2:] + n += int64(n2) + switch err { + case io.EOF: + js.filling = js.filling[:len(js.filling)-len(src)] + return n, nil + case nil: + default: + e.state.err = err + return n, err + } + if len(src) > 0 { + continue + } + if err = e.dispatchJob(false); err != nil { + return n, err + } + if cap(js.filling) < jobSize { + js.filling = make([]byte, 0, jobSize) + } + js.filling = js.filling[:jobSize] + src = js.filling + } +} + // Flush will send the currently written data to output // and block until everything has been written. // This should only be used on rare occasions where pushing the currently queued data is critical. func (e *Encoder) Flush() error { s := &e.state + if e.o.concurrentBlocks { + return e.flushJobs() + } if len(s.filling) > 0 { err := e.nextBlock(false) if err != nil { - // Ignore Flush after Close. if errors.Is(s.err, ErrEncoderClosed) { return nil } @@ -438,7 +561,6 @@ func (e *Encoder) Flush() error { s.wg.Wait() s.wWg.Wait() if s.err != nil { - // Ignore Flush after Close. if errors.Is(s.err, ErrEncoderClosed) { return nil } @@ -447,6 +569,20 @@ func (e *Encoder) Flush() error { return s.writeErr } +func (e *Encoder) flushJobs() error { + js := &e.state.jobs + if len(js.filling) > 0 { + if err := e.dispatchJob(false); err != nil { + return err + } + } + e.waitAllJobs() + js.mu.Lock() + fErr := js.flusherErr + js.mu.Unlock() + return fErr +} + // Close will flush the final output and close the stream. // The function will block until everything has been written. // The Encoder can still be re-used after calling this. @@ -455,12 +591,16 @@ func (e *Encoder) Close() error { if s.encoder == nil { return nil } + if e.o.concurrentBlocks { + return e.closeJobs() + } if s.w == nil { if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { return nil } return errors.New("zstd: encoder has no writer") } + err := e.nextBlock(true) if err != nil { if errors.Is(s.err, ErrEncoderClosed) { @@ -511,6 +651,68 @@ func (e *Encoder) Close() error { return s.err } +func (e *Encoder) closeJobs() error { + s := &e.state + js := &s.jobs + + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } + + if s.w == nil { + if len(js.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { + return nil + } + return errors.New("zstd: encoder has no writer") + } + + if err := e.dispatchJob(true); err != nil { + e.shutdownJobWorkers() + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } + return err + } + + if s.frameContentSize > 0 && s.nInput != s.frameContentSize { + e.shutdownJobWorkers() + return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput) + } + + if s.fullFrameWritten { + e.shutdownJobWorkers() + s.err = ErrEncoderClosed + return nil + } + + e.shutdownJobWorkers() + if js.flusherErr != nil { + return js.flusherErr + } + + // Write CRC + if e.o.crc { + var tmp [4]byte + _, s.err = s.w.Write(s.encoder.AppendCRC(tmp[:0])) + s.nWritten += 4 + } + + // Add padding + if s.err == nil && e.o.pad > 0 { + add := calcSkippableFrame(s.nWritten, int64(e.o.pad)) + frame, err := skippableFrame(js.filling[:0], add, rand.Reader) + if err != nil { + return err + } + _, s.err = s.w.Write(frame) + } + if s.err == nil { + s.err = ErrEncoderClosed + return nil + } + return s.err +} + // EncodeAll will encode all input in src and append it to dst. // This function can be called concurrently, but each call will only run on a single goroutine. // If empty input is given, nothing is returned, unless WithZeroFrames is specified. diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index e217be0a1..a80814967 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -14,22 +14,23 @@ type EOption func(*encoderOptions) error // options retains accumulated state of multiple options. type encoderOptions struct { - resetOpt bool - concurrent int - level EncoderLevel - single *bool - pad int - blockSize int - windowSize int - crc bool - fullZero bool - noEntropy bool - allLitEntropy bool - customWindow bool - customALEntropy bool - customBlockSize bool - lowMem bool - dict *dict + resetOpt bool + concurrent int + level EncoderLevel + single *bool + pad int + blockSize int + windowSize int + crc bool + fullZero bool + noEntropy bool + allLitEntropy bool + customWindow bool + customALEntropy bool + customBlockSize bool + lowMem bool + dict *dict + concurrentBlocks bool } func (o *encoderOptions) setDefault() { @@ -333,6 +334,42 @@ func WithLowerEncoderMem(b bool) EOption { } } +// WithConcurrentBlocks enables job-based parallel compression for streams. +// When enabled and concurrent > 1, input is split into large sections (jobs) +// that are compressed simultaneously by multiple goroutines. +// Each non-first job receives an overlap prefix from the previous job for match context. +// Output is flushed in order, producing a valid single-frame zstd stream. +// +// Currently disabled when used with dictionary encoding. +// Cannot be changed with ResetWithOptions. +func WithConcurrentBlocks(b bool) EOption { + return func(o *encoderOptions) error { + if o.resetOpt && b != o.concurrentBlocks { + return errors.New("WithConcurrentBlocks cannot be changed on Reset") + } + o.concurrentBlocks = b + return nil + } +} + +// jobSize returns the input section size per parallel job. +func (o *encoderOptions) jobSize() int { + s := max(o.windowSize*4, 512<<10) + return s +} + +// overlapSize returns the overlap prefix size for parallel jobs. +func (o *encoderOptions) overlapSize() int { + switch o.level { + case SpeedBestCompression: + return o.windowSize / 2 + case SpeedBetterCompression: + return o.windowSize / 4 + default: + return o.windowSize / 8 + } +} + // WithEncoderDict allows to register a dictionary that will be used for the encode. // // The slice dict must be in the [dictionary format] produced by diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s index bcde39869..deeadc49e 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s @@ -1,4 +1,4 @@ -// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. +// Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s new file mode 100644 index 000000000..77ee3913f --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -0,0 +1,153 @@ +// Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && !appengine && !noasm && gc && !noasm + +// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int +TEXT ·buildDtable_asm(SB), $0-24 + MOVD ctx+8(FP), R1 + MOVD s+0(FP), R6 + + // Load values + MOVBU 4098(R6), R2 + MOVD $0, R0 + MOVD $1, R16 + LSL R2, R16, R16 + ORR R16, R0, R0 + MOVD (R1), R3 + MOVD 16(R1), R5 + SUB $1, R0, R7 + MOVD 8(R1), R1 + MOVHU 4096(R6), R6 + + // End load values + // Init, lay down lowprob symbols + MOVD $0, R8 + JMP init_main_loop_condition + +init_main_loop: + ADD R8<<1, R1, R15 + MOVH (R15), R9 + AND $0xffff, R9, R15 + MOVD $-1, R16 + AND $0xffff, R16, R16 + CMP R16, R15 + BNE do_not_update_high_threshold + ADD R7<<3, R5, R15 + MOVB R8, 1(R15) + SUB $1, R7, R7 + MOVD $0x0000000000000001, R9 + +do_not_update_high_threshold: + ADD R8<<1, R3, R15 + MOVH R9, (R15) + ADD $1, R8, R8 + +init_main_loop_condition: + CMP R6, R8 + BLT init_main_loop + + // Spread symbols + // Calculate table step + MOVD R0, R8 + LSR $0x01, R8, R8 + MOVD R0, R9 + LSR $0x03, R9, R9 + ADD R9, R8, R8 + ADD $3, R8, R8 + + // Fill add bits values + SUB $1, R0, R9 + MOVD $0, R10 + MOVD $0, R11 + JMP spread_main_loop_condition + +spread_main_loop: + MOVD $0, R12 + ADD R11<<1, R1, R15 + MOVH (R15), R13 + JMP spread_inner_loop_condition + +spread_inner_loop: + ADD R10<<3, R5, R15 + MOVB R11, 1(R15) + +adjust_position: + ADD R8, R10, R10 + AND R9, R10, R10 + CMP R7, R10 + BGT adjust_position + ADD $1, R12, R12 + +spread_inner_loop_condition: + CMP R13, R12 + BLT spread_inner_loop + ADD $1, R11, R11 + +spread_main_loop_condition: + CMP R6, R11 + BLT spread_main_loop + TST R10, R10 + BEQ spread_check_ok + MOVD ctx+8(FP), R0 + MOVD R10, 24(R0) + MOVD $+1, R16 + MOVD R16, ret+16(FP) + RET + +spread_check_ok: + // Build Decoding table + MOVD $0, R6 + +build_table_main_table: + ADD R6<<3, R5, R15 + MOVBU 1(R15), R1 + ADD R1<<1, R3, R15 + MOVHU (R15), R7 + ADD $1, R7, R8 + ADD R1<<1, R3, R15 + MOVH R8, (R15) + MOVD R7, R8 + CLZ R8, R16 + MOVD $63, R8 + SUB R16, R8, R8 + MOVD R2, R1 + SUB R8, R1, R1 + LSL R1, R7, R7 + SUB R0, R7, R7 + ADD R6<<3, R5, R15 + MOVB R1, (R15) + ADD R6<<3, R5, R15 + MOVH R7, 2(R15) + CMP R0, R7 + BLE build_table_check1_ok + MOVD ctx+8(FP), R1 + MOVD R7, 24(R1) + MOVD R0, 32(R1) + MOVD $+2, R16 + MOVD R16, ret+16(FP) + RET + +build_table_check1_ok: + AND $0xff, R1, R15 + AND $0xff, R1, R16 + TST R16, R15 + BNE build_table_check2_ok + AND $0xffff, R7, R15 + AND $0xffff, R6, R16 + CMP R16, R15 + BNE build_table_check2_ok + MOVD ctx+8(FP), R0 + MOVD R7, 24(R0) + MOVD R6, 32(R0) + MOVD $+3, R16 + MOVD R16, ret+16(FP) + RET + +build_table_check2_ok: + ADD $1, R6, R6 + CMP R0, R6 + BLT build_table_main_table + MOVD $+0, R16 + MOVD R16, ret+16(FP) + RET diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go similarity index 81% rename from vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go rename to vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go index b8c8607b5..4ffc7e3c9 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go @@ -1,4 +1,4 @@ -//go:build amd64 && !appengine && !noasm && gc +//go:build (amd64 || arm64) && !appengine && !noasm && gc package zstd @@ -6,6 +6,10 @@ import ( "fmt" ) +// buildDtable_asm is generated by _generate/gen_fse.go and lowered to each +// architecture (amd64 by goasm, arm64 by the avo arm64 lowering printer). The +// Go side is identical across architectures, so it lives here. + type buildDtableAsmContext struct { // inputs stateTable *uint16 @@ -18,7 +22,7 @@ type buildDtableAsmContext struct { errParam2 uint64 } -// buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable. +// buildDtable_asm is an assembly implementation of fseDecoder.buildDtable. // Function returns non-zero exit code on error. // //go:noescape diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 2138f8091..38fd2ccb2 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count deleted file mode 100644 index 07405d273..000000000 Binary files a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count and /dev/null differ diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/weekends b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/weekends deleted file mode 100644 index b8626c4cf..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/~/Library/Application Support/go/telemetry/local/weekends +++ /dev/null @@ -1 +0,0 @@ -4 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 18c3703dd..1281da885 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -3,321 +3,83 @@ package zstd import ( - "fmt" - "io" - "github.com/klauspost/compress/internal/cpuinfo" ) -type decodeSyncAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - litRemain int - out []byte - outPosition int - literals []byte - litPosition int - history []byte - windowSize int - ll int // set on error (not for all errors, please refer to _generate/gen.go) - ml int // set on error (not for all errors, please refer to _generate/gen.go) - mo int // set on error (not for all errors, please refer to _generate/gen.go) -} +// The shared decode/decodeSync/executeSimple wrappers and context structs live +// in seqdec_asm.go; this file only declares the amd64 asm routines and the +// dispatch helpers that pick the BMI2 / non-BMI2 (and 56-bit / safe) variant. -// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. // //go:noescape -func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. +// sequenceDecs_decode_56_amd64 implements the main loop of sequenceDecs in x86 asm. // //go:noescape -func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. +// sequenceDecs_decode_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. +// sequenceDecs_decode_56_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// decode sequences from the stream with the provided history but without a dictionary. -func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { - if len(s.dict) > 0 { - return false, nil - } - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { - return false, nil - } - - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true - /* - useSafe := false - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { - useSafe = true - } - if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { - useSafe = true - } - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - useSafe = true - } - */ - - br := s.br - - maxBlockSize := min(s.windowSize, maxCompressedBlockSize) - - ctx := decodeSyncAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - iteration: s.nSeqs - 1, - litRemain: len(s.literals), - out: s.out, - outPosition: len(s.out), - literals: s.literals, - windowSize: s.windowSize, - history: hist, - } - - s.seqSize = 0 - startSize := len(s.out) +func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - var errCode int +// decodeAsm runs the sequenceDecs decode loop, choosing the BMI2 / 56-bit variant. +func decodeAsm(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext, lte56bits bool) int { if cpuinfo.HasBMI2() { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_bmi2(s, br, &ctx) - } - } else { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_amd64(s, br, &ctx) - } - } - switch errCode { - case noError: - break - - case errorMatchLenOfsMismatch: - return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml) - - case errorMatchLenTooBig: - return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml) - - case errorMatchOffTooBig: - return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", - ctx.mo, ctx.outPosition+len(hist)-startSize) - - case errorNotEnoughLiterals: - return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", - ctx.ll, ctx.litRemain+ctx.ll) - - case errorOverread: - return true, io.ErrUnexpectedEOF - - case errorNotEnoughSpace: - size := ctx.outPosition + ctx.ll + ctx.ml - if debugDecoder { - println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize) + if lte56bits { + return sequenceDecs_decode_56_bmi2(s, br, ctx) } - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - - default: - return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) - } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + return sequenceDecs_decode_bmi2(s, br, ctx) } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - return true, err + if lte56bits { + return sequenceDecs_decode_56_amd64(s, br, ctx) } - - s.literals = s.literals[ctx.litPosition:] - t := ctx.outPosition - s.out = s.out[:t] - - // Add final literals - s.out = append(s.out, s.literals...) - if debugDecoder { - t += len(s.literals) - if t != len(s.out) { - panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t)) - } - } - - return true, nil + return sequenceDecs_decode_amd64(s, br, ctx) } -// -------------------------------------------------------------------------------- - -type decodeAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - seqs []seqVals - litRemain int -} - -const noError = 0 - -// error reported when mo == 0 && ml > 0 -const errorMatchLenOfsMismatch = 1 - -// error reported when ml > maxMatchLen -const errorMatchLenTooBig = 2 - -// error reported when mo > available history or mo > s.windowSize -const errorMatchOffTooBig = 3 - -// error reported when the sum of literal lengths exeeceds the literal buffer size -const errorNotEnoughLiterals = 4 - -// error reported when capacity of `out` is too small -const errorNotEnoughSpace = 5 - -// error reported when bits are overread. -const errorOverread = 6 - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. +// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. // //go:noescape -func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. -// -// Please refer to seqdec_generic.go for the reference implementation. +// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. // //go:noescape -func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. // //go:noescape -func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// decode sequences from the stream without the provided history. -func (s *sequenceDecs) decode(seqs []seqVals) error { - br := s.br - - maxBlockSize := min(s.windowSize, maxCompressedBlockSize) - - ctx := decodeAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - seqs: seqs, - iteration: len(seqs) - 1, - litRemain: len(s.literals), - } - - if debugDecoder { - println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream") - } +func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - s.seqSize = 0 - lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 - var errCode int +// decodeSyncAsm runs the decodeSync loop, choosing the BMI2 / safe variant. +func decodeSyncAsm(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext, safe bool) int { if cpuinfo.HasBMI2() { - if lte56bits { - errCode = sequenceDecs_decode_56_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_bmi2(s, br, &ctx) - } - } else { - if lte56bits { - errCode = sequenceDecs_decode_56_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_amd64(s, br, &ctx) + if safe { + return sequenceDecs_decodeSync_safe_bmi2(s, br, ctx) } + return sequenceDecs_decodeSync_bmi2(s, br, ctx) } - if errCode != 0 { - i := len(seqs) - ctx.iteration - 1 - switch errCode { - case errorMatchLenOfsMismatch: - ml := ctx.seqs[i].ml - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - - case errorMatchLenTooBig: - ml := ctx.seqs[i].ml - return fmt.Errorf("match len (%d) bigger than max allowed length", ml) - - case errorNotEnoughLiterals: - ll := ctx.seqs[i].ll - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) - case errorOverread: - return io.ErrUnexpectedEOF - } - - return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) + if safe { + return sequenceDecs_decodeSync_safe_amd64(s, br, ctx) } - - if ctx.litRemain < 0 { - return fmt.Errorf("literal count is too big: total available %d, total requested %d", - len(s.literals), len(s.literals)-ctx.litRemain) - } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - if debugDecoder { - println("decode: ", br.remain(), "bits remain on stream. code:", errCode) - } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - } - return err -} - -// -------------------------------------------------------------------------------- - -type executeAsmContext struct { - seqs []seqVals - seqIndex int - out []byte - history []byte - literals []byte - outPosition int - litPosition int - windowSize int + return sequenceDecs_decodeSync_amd64(s, br, ctx) } // sequenceDecs_executeSimple_amd64 implements the main loop of sequenceDecs.executeSimple in x86 asm. @@ -334,54 +96,10 @@ func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool //go:noescape func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool -// executeSimple handles cases when dictionary is not used. -func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { - // Ensure we have enough output size... - if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { - addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] - } - - if debugDecoder { - printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) - } - - var t = len(s.out) - out := s.out[:t+s.seqSize] - - ctx := executeAsmContext{ - seqs: seqs, - seqIndex: 0, - out: out, - history: hist, - outPosition: t, - litPosition: 0, - literals: s.literals, - windowSize: s.windowSize, +// executeSimpleAsm runs the executeSimple loop, choosing the safe variant. +func executeSimpleAsm(ctx *executeAsmContext, safe bool) bool { + if safe { + return sequenceDecs_executeSimple_safe_amd64(ctx) } - var ok bool - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - ok = sequenceDecs_executeSimple_safe_amd64(&ctx) - } else { - ok = sequenceDecs_executeSimple_amd64(&ctx) - } - if !ok { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", - seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) - } - s.literals = s.literals[ctx.litPosition:] - t = ctx.outPosition - - // Add final literals - copy(out[t:], s.literals) - if debugDecoder { - t += len(s.literals) - if t != len(out) { - panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) - } - } - s.out = out - - return nil + return sequenceDecs_executeSimple_amd64(ctx) } diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index a708ca6d3..4d3188ff4 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1,4 +1,4 @@ -// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm @@ -2081,7 +2081,7 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ 16(AX)(R13*1), R14 ADDQ R10, R14 CMPQ R14, 32(SP) JA error_not_enough_space @@ -2601,7 +2601,7 @@ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ 16(CX)(R13*1), R14 ADDQ R9, R14 CMPQ R14, 32(SP) JA error_not_enough_space diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go new file mode 100644 index 000000000..5ad262acf --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go @@ -0,0 +1,70 @@ +//go:build arm64 && !appengine && !noasm && gc + +package zstd + +// The shared decode/decodeSync/executeSimple wrappers and context structs live +// in seqdec_asm.go; this file only declares the arm64 asm routines (generated +// by the avo arm64 lowering printer) and the dispatch helpers. arm64 has no +// BMI2, so each helper selects only between the 56-bit / safe variants. + +// sequenceDecs_decode_arm64 implements the main loop of sequenceDecs in arm64 asm. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_decode_arm64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// sequenceDecs_decode_56_arm64 implements the main loop of sequenceDecs in arm64 asm. +// +//go:noescape +func sequenceDecs_decode_56_arm64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// decodeAsm runs the sequenceDecs decode loop, choosing the 56-bit variant. +func decodeAsm(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext, lte56bits bool) int { + if lte56bits { + return sequenceDecs_decode_56_arm64(s, br, ctx) + } + return sequenceDecs_decode_arm64(s, br, ctx) +} + +// sequenceDecs_decodeSync_arm64 implements the main loop of sequenceDecs.decodeSync in arm64 asm. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_decodeSync_arm64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// sequenceDecs_decodeSync_safe_arm64 does the same as above, but does not write more than output buffer. +// +//go:noescape +func sequenceDecs_decodeSync_safe_arm64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// decodeSyncAsm runs the decodeSync loop, choosing the safe variant. +func decodeSyncAsm(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext, safe bool) int { + if safe { + return sequenceDecs_decodeSync_safe_arm64(s, br, ctx) + } + return sequenceDecs_decodeSync_arm64(s, br, ctx) +} + +// sequenceDecs_executeSimple_arm64 implements the main loop of sequenceDecs.executeSimple in arm64 asm. +// +// Returns false if a match offset is too big. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_executeSimple_arm64(ctx *executeAsmContext) bool + +// Same as above, but with safe memcopies +// +//go:noescape +func sequenceDecs_executeSimple_safe_arm64(ctx *executeAsmContext) bool + +// executeSimpleAsm runs the executeSimple loop, choosing the safe variant. +func executeSimpleAsm(ctx *executeAsmContext, safe bool) bool { + if safe { + return sequenceDecs_executeSimple_safe_arm64(ctx) + } + return sequenceDecs_executeSimple_arm64(ctx) +} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s new file mode 100644 index 000000000..6f54ef21b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -0,0 +1,2750 @@ +// Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && !appengine && !noasm && gc && !noasm + +// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: CMOV +TEXT ·sequenceDecs_decode_arm64(SB), $8-32 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD 104(R0), R9 + MOVD s+0(FP), R0 + MOVD 144(R0), R10 + MOVD 152(R0), R11 + MOVD 160(R0), R12 + +sequenceDecs_decode_amd64_main_loop: + MOVD 8(RSP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMP $0x08, R5 + BLT sequenceDecs_decode_amd64_fill_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R13, R13 + MOVD (R13), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decode_amd64_fill_end + +sequenceDecs_decode_amd64_fill_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decode_amd64_fill_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decode_amd64_fill_end + LSL $0x08, R2, R2 + SUB $0x01, R13, R13 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R13), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decode_amd64_fill_byte_by_byte + +sequenceDecs_decode_amd64_fill_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decode_amd64_fill_end: + // Update offset + MOVD R8, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_amd64_of_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_amd64_of_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_amd64_of_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_amd64_of_update_zero: + MOVD R0, 16(R9) + + // Update match length + MOVD R7, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_amd64_ml_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_amd64_ml_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_amd64_ml_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_amd64_ml_update_zero: + MOVD R0, 8(R9) + + // Fill bitreader to have enough for the remaining + CMP $0x08, R5 + BLT sequenceDecs_decode_amd64_fill_2_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R13, R13 + MOVD (R13), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decode_amd64_fill_2_end + +sequenceDecs_decode_amd64_fill_2_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decode_amd64_fill_2_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decode_amd64_fill_2_end + LSL $0x08, R2, R2 + SUB $0x01, R13, R13 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R13), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decode_amd64_fill_2_byte_by_byte + +sequenceDecs_decode_amd64_fill_2_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decode_amd64_fill_2_end: + // Update literal length + MOVD R6, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_amd64_ll_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_amd64_ll_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_amd64_ll_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_amd64_ll_update_zero: + MOVD R0, (R9) + + // Fill bitreader for state updates + MOVD R13, 8(RSP) + MOVD R8, R0 + LSR $0x08, R0, R0 + MOVBU R0, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_amd64_skip_update + + // Update Literal Length State + MOVBU R6, R13 + LSRW $0x10, R6, R6 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R6, R6 + + // Load ctx.llTable + MOVD ctx+16(FP), R1 + MOVD (R1), R1 + ADD R6<<3, R1, R15 + MOVD (R15), R6 + + // Update Match Length State + MOVBU R7, R13 + LSRW $0x10, R7, R7 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R7, R7 + + // Load ctx.mlTable + MOVD ctx+16(FP), R1 + MOVD 24(R1), R1 + ADD R7<<3, R1, R15 + MOVD (R15), R7 + + // Update Offset State + MOVBU R8, R13 + LSRW $0x10, R8, R8 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R8, R8 + + // Load ctx.ofTable + MOVD ctx+16(FP), R1 + MOVD 48(R1), R1 + ADD R8<<3, R1, R15 + MOVD (R15), R8 + +sequenceDecs_decode_amd64_skip_update: + // Adjust offset + MOVD 16(R9), R1 + CMP $0x01, R0 + BLS sequenceDecs_decode_amd64_adjust_offsetB_1_or_0 + MOVD R11, R12 + MOVD R10, R11 + MOVD R1, R10 + JMP sequenceDecs_decode_amd64_after_adjust + +sequenceDecs_decode_amd64_adjust_offsetB_1_or_0: + MOVD (R9), R16 + CMP $0x00000000, R16 + BNE sequenceDecs_decode_amd64_adjust_offset_maybezero + ADD $1, R1, R1 + JMP sequenceDecs_decode_amd64_adjust_offset_nonzero + +sequenceDecs_decode_amd64_adjust_offset_maybezero: + TST R1, R1 + BNE sequenceDecs_decode_amd64_adjust_offset_nonzero + MOVD R10, R1 + JMP sequenceDecs_decode_amd64_after_adjust + +sequenceDecs_decode_amd64_adjust_offset_nonzero: + CMP $0x01, R1 + BLO sequenceDecs_decode_amd64_adjust_zero + BEQ sequenceDecs_decode_amd64_adjust_one + CMP $0x02, R1 + BHI sequenceDecs_decode_amd64_adjust_three + JMP sequenceDecs_decode_amd64_adjust_two + +sequenceDecs_decode_amd64_adjust_zero: + MOVD R10, R0 + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_one: + MOVD R11, R0 + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_two: + MOVD R12, R0 + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_three: + SUB $1, R10, R0 + +sequenceDecs_decode_amd64_adjust_test_temp_valid: + TST R0, R0 + BNE sequenceDecs_decode_amd64_adjust_temp_valid + MOVD $0x00000001, R0 + +sequenceDecs_decode_amd64_adjust_temp_valid: + CMP $0x01, R1 + CSEL NE, R11, R12, R12 + MOVD R10, R11 + MOVD R0, R10 + MOVD R0, R1 + +sequenceDecs_decode_amd64_after_adjust: + MOVD R1, 16(R9) + + // Check values + MOVD 8(R9), R0 + MOVD (R9), R13 + ADD R13, R0, R14 + MOVD s+0(FP), R4 + MOVD 256(R4), R16 + ADD R14, R16, R16 + MOVD R16, 256(R4) + MOVD ctx+16(FP), R14 + MOVD 128(R14), R16 + SUBS R13, R16, R16 + MOVD R16, 128(R14) + BMI error_not_enough_literals + CMP $0x00020002, R0 + BHI sequenceDecs_decode_amd64_error_match_len_too_big + TST R1, R1 + BNE sequenceDecs_decode_amd64_match_len_ofs_ok + TST R0, R0 + BNE sequenceDecs_decode_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decode_amd64_match_len_ofs_ok: + ADD $0x18, R9, R9 + MOVD ctx+16(FP), R0 + MOVD 96(R0), R16 + SUBS $1, R16, R16 + MOVD R16, 96(R0) + BPL sequenceDecs_decode_amd64_main_loop + MOVD s+0(FP), R0 + MOVD R10, 144(R0) + MOVD R11, 152(R0) + MOVD R12, 160(R0) + MOVD br+8(FP), R0 + MOVD R2, 24(R0) + MOVB R3, 40(R0) + MOVD R5, 32(R0) + + // Return success + MOVD $0x00000000, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVD $0x00000001, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_amd64_error_match_len_too_big: + MOVD $0x00000002, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match offset too long error + MOVD $0x00000003, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVD $0x00000004, R16 + MOVD R16, ret+24(FP) + RET + + // Return with overread error +error_overread: + MOVD $0x00000006, R16 + MOVD R16, ret+24(FP) + RET + +// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: CMOV +TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD 104(R0), R9 + MOVD s+0(FP), R0 + MOVD 144(R0), R10 + MOVD 152(R0), R11 + MOVD 160(R0), R12 + +sequenceDecs_decode_56_amd64_main_loop: + MOVD 8(RSP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMP $0x08, R5 + BLT sequenceDecs_decode_56_amd64_fill_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R13, R13 + MOVD (R13), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decode_56_amd64_fill_end + +sequenceDecs_decode_56_amd64_fill_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decode_56_amd64_fill_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decode_56_amd64_fill_end + LSL $0x08, R2, R2 + SUB $0x01, R13, R13 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R13), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decode_56_amd64_fill_byte_by_byte + +sequenceDecs_decode_56_amd64_fill_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decode_56_amd64_fill_end: + // Update offset + MOVD R8, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_56_amd64_of_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_56_amd64_of_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_56_amd64_of_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_56_amd64_of_update_zero: + MOVD R0, 16(R9) + + // Update match length + MOVD R7, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_56_amd64_ml_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_56_amd64_ml_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_56_amd64_ml_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_56_amd64_ml_update_zero: + MOVD R0, 8(R9) + + // Update literal length + MOVD R6, R0 + MOVD R3, R1 + MOVD R2, R14 + LSL R1, R14, R14 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decode_56_amd64_ll_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decode_56_amd64_ll_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decode_56_amd64_ll_update_zero + NEG R1, R1 + LSR R1, R14, R14 + ADD R14, R0, R0 + +sequenceDecs_decode_56_amd64_ll_update_zero: + MOVD R0, (R9) + + // Fill bitreader for state updates + MOVD R13, 8(RSP) + MOVD R8, R0 + LSR $0x08, R0, R0 + MOVBU R0, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_56_amd64_skip_update + + // Update Literal Length State + MOVBU R6, R13 + LSRW $0x10, R6, R6 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R6, R6 + + // Load ctx.llTable + MOVD ctx+16(FP), R1 + MOVD (R1), R1 + ADD R6<<3, R1, R15 + MOVD (R15), R6 + + // Update Match Length State + MOVBU R7, R13 + LSRW $0x10, R7, R7 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R7, R7 + + // Load ctx.mlTable + MOVD ctx+16(FP), R1 + MOVD 24(R1), R1 + ADD R7<<3, R1, R15 + MOVD (R15), R7 + + // Update Offset State + MOVBU R8, R13 + LSRW $0x10, R8, R8 + ADD R13, R3, R1 + MOVD R2, R14 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R14, R14 + MOVD $0x00000001, R4 + BFI $0, R13, $8, R1 + LSLW R1, R4, R4 + SUBW $1, R4, R4 + AND R4, R14, R14 + ADD R14, R8, R8 + + // Load ctx.ofTable + MOVD ctx+16(FP), R1 + MOVD 48(R1), R1 + ADD R8<<3, R1, R15 + MOVD (R15), R8 + +sequenceDecs_decode_56_amd64_skip_update: + // Adjust offset + MOVD 16(R9), R1 + CMP $0x01, R0 + BLS sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0 + MOVD R11, R12 + MOVD R10, R11 + MOVD R1, R10 + JMP sequenceDecs_decode_56_amd64_after_adjust + +sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0: + MOVD (R9), R16 + CMP $0x00000000, R16 + BNE sequenceDecs_decode_56_amd64_adjust_offset_maybezero + ADD $1, R1, R1 + JMP sequenceDecs_decode_56_amd64_adjust_offset_nonzero + +sequenceDecs_decode_56_amd64_adjust_offset_maybezero: + TST R1, R1 + BNE sequenceDecs_decode_56_amd64_adjust_offset_nonzero + MOVD R10, R1 + JMP sequenceDecs_decode_56_amd64_after_adjust + +sequenceDecs_decode_56_amd64_adjust_offset_nonzero: + CMP $0x01, R1 + BLO sequenceDecs_decode_56_amd64_adjust_zero + BEQ sequenceDecs_decode_56_amd64_adjust_one + CMP $0x02, R1 + BHI sequenceDecs_decode_56_amd64_adjust_three + JMP sequenceDecs_decode_56_amd64_adjust_two + +sequenceDecs_decode_56_amd64_adjust_zero: + MOVD R10, R0 + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_one: + MOVD R11, R0 + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_two: + MOVD R12, R0 + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_three: + SUB $1, R10, R0 + +sequenceDecs_decode_56_amd64_adjust_test_temp_valid: + TST R0, R0 + BNE sequenceDecs_decode_56_amd64_adjust_temp_valid + MOVD $0x00000001, R0 + +sequenceDecs_decode_56_amd64_adjust_temp_valid: + CMP $0x01, R1 + CSEL NE, R11, R12, R12 + MOVD R10, R11 + MOVD R0, R10 + MOVD R0, R1 + +sequenceDecs_decode_56_amd64_after_adjust: + MOVD R1, 16(R9) + + // Check values + MOVD 8(R9), R0 + MOVD (R9), R13 + ADD R13, R0, R14 + MOVD s+0(FP), R4 + MOVD 256(R4), R16 + ADD R14, R16, R16 + MOVD R16, 256(R4) + MOVD ctx+16(FP), R14 + MOVD 128(R14), R16 + SUBS R13, R16, R16 + MOVD R16, 128(R14) + BMI error_not_enough_literals + CMP $0x00020002, R0 + BHI sequenceDecs_decode_56_amd64_error_match_len_too_big + TST R1, R1 + BNE sequenceDecs_decode_56_amd64_match_len_ofs_ok + TST R0, R0 + BNE sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decode_56_amd64_match_len_ofs_ok: + ADD $0x18, R9, R9 + MOVD ctx+16(FP), R0 + MOVD 96(R0), R16 + SUBS $1, R16, R16 + MOVD R16, 96(R0) + BPL sequenceDecs_decode_56_amd64_main_loop + MOVD s+0(FP), R0 + MOVD R10, 144(R0) + MOVD R11, 152(R0) + MOVD R12, 160(R0) + MOVD br+8(FP), R0 + MOVD R2, 24(R0) + MOVB R3, 40(R0) + MOVD R5, 32(R0) + + // Return success + MOVD $0x00000000, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVD $0x00000001, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVD $0x00000002, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match offset too long error + MOVD $0x00000003, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVD $0x00000004, R16 + MOVD R16, ret+24(FP) + RET + + // Return with overread error +error_overread: + MOVD $0x00000006, R16 + MOVD R16, ret+24(FP) + RET + +// skipped sequenceDecs_decode_bmi2 (generic twin preferred on arm64) + +// skipped sequenceDecs_decode_56_bmi2 (generic twin preferred on arm64) + +// func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool +// Requires: SSE +TEXT ·sequenceDecs_executeSimple_arm64(SB), $8-9 + MOVD ctx+0(FP), R9 + MOVD 8(R9), R1 + TST R1, R1 + BEQ empty_seqs + MOVD (R9), R0 + MOVD 24(R9), R2 + MOVD 32(R9), R3 + MOVD 80(R9), R5 + MOVD 104(R9), R6 + MOVD 120(R9), R7 + MOVD 56(R9), R8 + MOVD 64(R9), R9 + ADD R9, R8, R8 + + // seqsBase += 24 * seqIndex + ADD R2<<1, R2, R10 + LSL $0x03, R10, R10 + ADD R10, R0, R0 + + // outBase += outPosition + ADD R6, R3, R3 + +main_loop: + MOVD (R0), R10 + MOVD 16(R0), R11 + MOVD 8(R0), R12 + + // Copy literals + TST R10, R10 + BEQ check_offset + MOVD $0, R13 + +copy_1: + ADD R13, R5, R15 + VLD1 (R15), [V0.B16] + ADD R13, R3, R15 + VST1 [V0.B16], (R15) + ADD $0x10, R13, R13 + CMP R10, R13 + BLO copy_1 + ADD R10, R5, R5 + ADD R10, R3, R3 + ADD R10, R6, R6 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + ADD R9, R6, R10 + CMP R10, R11 + BGT error_match_off_too_big + CMP R7, R11 + BGT error_match_off_too_big + + // Copy match from history + MOVD R11, R10 + SUBS R6, R10, R10 + BLS copy_match + MOVD R8, R13 + SUB R10, R13, R13 + CMP R10, R12 + BGT copy_all_from_history + MOVD R12, R10 + SUBS $0x10, R10, R10 + BLO copy_4_small + +copy_4_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_4_end + +copy_4_small: + CMP $0x03, R12 + BEQ copy_4_move_3 + CMP $0x08, R12 + BLO copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end + +copy_4_move_4through7: + MOVWU (R13), R10 + ADD R12, R13, R15 + MOVWU -4(R15), R11 + MOVW R10, (R3) + ADD R12, R3, R15 + MOVW R11, -4(R15) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end + +copy_4_move_8through16: + MOVD (R13), R10 + ADD R12, R13, R15 + MOVD -8(R15), R11 + MOVD R10, (R3) + ADD R12, R3, R15 + MOVD R11, -8(R15) + ADD R12, R13, R13 + ADD R12, R3, R3 + +copy_4_end: + ADD R12, R6, R6 + ADD $0x18, R0, R0 + ADD $1, R2, R2 + CMP R1, R2 + BLO main_loop + JMP loop_finished + +copy_all_from_history: + MOVD R10, R14 + SUBS $0x10, R14, R14 + BLO copy_5_small + +copy_5_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_5_end + +copy_5_small: + CMP $0x03, R10 + BEQ copy_5_move_3 + BLO copy_5_move_1or2 + CMP $0x08, R10 + BLO copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_3: + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_4through7: + MOVWU (R13), R14 + ADD R10, R13, R15 + MOVWU -4(R15), R4 + MOVW R14, (R3) + ADD R10, R3, R15 + MOVW R4, -4(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_8through16: + MOVD (R13), R14 + ADD R10, R13, R15 + MOVD -8(R15), R4 + MOVD R14, (R3) + ADD R10, R3, R15 + MOVD R4, -8(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + +copy_5_end: + ADD R10, R6, R6 + SUB R10, R12, R12 + + // Copy match from the current buffer +copy_match: + MOVD R3, R10 + SUB R11, R10, R10 + + // ml <= mo + CMP R11, R12 + BHI copy_overlapping_match + + // Copy non-overlapping match + ADD R12, R6, R6 + MOVD R3, R11 + ADD R12, R3, R3 + +copy_2: + VLD1 (R10), [V0.B16] + VST1 [V0.B16], (R11) + ADD $0x10, R10, R10 + ADD $0x10, R11, R11 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADD R12, R6, R6 + +copy_slow_3: + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 + +handle_loop: + ADD $0x18, R0, R0 + ADD $1, R2, R2 + CMP R1, R2 + BLO main_loop + +loop_finished: + // Return value + MOVD $0x01, R16 + MOVB R16, ret+8(FP) + + // Update the context + MOVD ctx+0(FP), R0 + MOVD R2, 24(R0) + MOVD R6, 104(R0) + MOVD 80(R0), R16 + SUB R16, R5, R5 + MOVD R5, 112(R0) + RET + +error_match_off_too_big: + // Return value + MOVD $0x00, R16 + MOVB R16, ret+8(FP) + + // Update the context + MOVD ctx+0(FP), R0 + MOVD R2, 24(R0) + MOVD R6, 104(R0) + MOVD 80(R0), R16 + SUB R16, R5, R5 + MOVD R5, 112(R0) + RET + +empty_seqs: + // Return value + MOVD $0x01, R16 + MOVB R16, ret+8(FP) + RET + +// func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool +// Requires: SSE +TEXT ·sequenceDecs_executeSimple_safe_arm64(SB), $8-9 + MOVD ctx+0(FP), R9 + MOVD 8(R9), R1 + TST R1, R1 + BEQ empty_seqs + MOVD (R9), R0 + MOVD 24(R9), R2 + MOVD 32(R9), R3 + MOVD 80(R9), R5 + MOVD 104(R9), R6 + MOVD 120(R9), R7 + MOVD 56(R9), R8 + MOVD 64(R9), R9 + ADD R9, R8, R8 + + // seqsBase += 24 * seqIndex + ADD R2<<1, R2, R10 + LSL $0x03, R10, R10 + ADD R10, R0, R0 + + // outBase += outPosition + ADD R6, R3, R3 + +main_loop: + MOVD (R0), R10 + MOVD 16(R0), R11 + MOVD 8(R0), R12 + + // Copy literals + TST R10, R10 + BEQ check_offset + MOVD R10, R13 + SUBS $0x10, R13, R13 + BLO copy_1_small + +copy_1_loop: + VLD1 (R5), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R5, R5 + ADD $0x10, R3, R3 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R5, R5 + ADD $16, R5, R5 + ADD R13, R3, R3 + ADD $16, R3, R3 + ADD $-16, R5, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_1_end + +copy_1_small: + CMP $0x03, R10 + BEQ copy_1_move_3 + BLO copy_1_move_1or2 + CMP $0x08, R10 + BLO copy_1_move_4through7 + JMP copy_1_move_8through16 + +copy_1_move_1or2: + MOVBU (R5), R16 + BFI $0, R16, $8, R13 + ADD R10, R5, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R3) + ADD R10, R3, R15 + MOVB R14, -1(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end + +copy_1_move_3: + MOVHU (R5), R13 + MOVBU 2(R5), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R3) + MOVB R14, 2(R3) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end + +copy_1_move_4through7: + MOVWU (R5), R13 + ADD R10, R5, R15 + MOVWU -4(R15), R14 + MOVW R13, (R3) + ADD R10, R3, R15 + MOVW R14, -4(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end + +copy_1_move_8through16: + MOVD (R5), R13 + ADD R10, R5, R15 + MOVD -8(R15), R14 + MOVD R13, (R3) + ADD R10, R3, R15 + MOVD R14, -8(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + +copy_1_end: + ADD R10, R6, R6 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + ADD R9, R6, R10 + CMP R10, R11 + BGT error_match_off_too_big + CMP R7, R11 + BGT error_match_off_too_big + + // Copy match from history + MOVD R11, R10 + SUBS R6, R10, R10 + BLS copy_match + MOVD R8, R13 + SUB R10, R13, R13 + CMP R10, R12 + BGT copy_all_from_history + MOVD R12, R10 + SUBS $0x10, R10, R10 + BLO copy_4_small + +copy_4_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_4_end + +copy_4_small: + CMP $0x03, R12 + BEQ copy_4_move_3 + CMP $0x08, R12 + BLO copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end + +copy_4_move_4through7: + MOVWU (R13), R10 + ADD R12, R13, R15 + MOVWU -4(R15), R11 + MOVW R10, (R3) + ADD R12, R3, R15 + MOVW R11, -4(R15) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end + +copy_4_move_8through16: + MOVD (R13), R10 + ADD R12, R13, R15 + MOVD -8(R15), R11 + MOVD R10, (R3) + ADD R12, R3, R15 + MOVD R11, -8(R15) + ADD R12, R13, R13 + ADD R12, R3, R3 + +copy_4_end: + ADD R12, R6, R6 + ADD $0x18, R0, R0 + ADD $1, R2, R2 + CMP R1, R2 + BLO main_loop + JMP loop_finished + +copy_all_from_history: + MOVD R10, R14 + SUBS $0x10, R14, R14 + BLO copy_5_small + +copy_5_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_5_end + +copy_5_small: + CMP $0x03, R10 + BEQ copy_5_move_3 + BLO copy_5_move_1or2 + CMP $0x08, R10 + BLO copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_3: + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_4through7: + MOVWU (R13), R14 + ADD R10, R13, R15 + MOVWU -4(R15), R4 + MOVW R14, (R3) + ADD R10, R3, R15 + MOVW R4, -4(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end + +copy_5_move_8through16: + MOVD (R13), R14 + ADD R10, R13, R15 + MOVD -8(R15), R4 + MOVD R14, (R3) + ADD R10, R3, R15 + MOVD R4, -8(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + +copy_5_end: + ADD R10, R6, R6 + SUB R10, R12, R12 + + // Copy match from the current buffer +copy_match: + MOVD R3, R10 + SUB R11, R10, R10 + + // ml <= mo + CMP R11, R12 + BHI copy_overlapping_match + + // Copy non-overlapping match + ADD R12, R6, R6 + MOVD R12, R11 + SUBS $0x10, R11, R11 + BLO copy_2_small + +copy_2_loop: + VLD1 (R10), [V0.B16] + VST1 [V0.B16], (R3) + ADD $0x10, R10, R10 + ADD $0x10, R3, R3 + SUBS $0x10, R11, R11 + BHS copy_2_loop + ADD R11, R10, R10 + ADD $16, R10, R10 + ADD R11, R3, R3 + ADD $16, R3, R3 + ADD $-16, R10, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R3, R15 + VST1 [V0.B16], (R15) + JMP copy_2_end + +copy_2_small: + CMP $0x03, R12 + BEQ copy_2_move_3 + BLO copy_2_move_1or2 + CMP $0x08, R12 + BLO copy_2_move_4through7 + JMP copy_2_move_8through16 + +copy_2_move_1or2: + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + ADD R12, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R11, (R3) + ADD R12, R3, R15 + MOVB R13, -1(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end + +copy_2_move_3: + MOVHU (R10), R11 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R13 + MOVH R11, (R3) + MOVB R13, 2(R3) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end + +copy_2_move_4through7: + MOVWU (R10), R11 + ADD R12, R10, R15 + MOVWU -4(R15), R13 + MOVW R11, (R3) + ADD R12, R3, R15 + MOVW R13, -4(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end + +copy_2_move_8through16: + MOVD (R10), R11 + ADD R12, R10, R15 + MOVD -8(R15), R13 + MOVD R11, (R3) + ADD R12, R3, R15 + MOVD R13, -8(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + +copy_2_end: + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADD R12, R6, R6 + +copy_slow_3: + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 + +handle_loop: + ADD $0x18, R0, R0 + ADD $1, R2, R2 + CMP R1, R2 + BLO main_loop + +loop_finished: + // Return value + MOVD $0x01, R16 + MOVB R16, ret+8(FP) + + // Update the context + MOVD ctx+0(FP), R0 + MOVD R2, 24(R0) + MOVD R6, 104(R0) + MOVD 80(R0), R16 + SUB R16, R5, R5 + MOVD R5, 112(R0) + RET + +error_match_off_too_big: + // Return value + MOVD $0x00, R16 + MOVB R16, ret+8(FP) + + // Update the context + MOVD ctx+0(FP), R0 + MOVD R2, 24(R0) + MOVD R6, 104(R0) + MOVD 80(R0), R16 + SUB R16, R5, R5 + MOVD R5, 112(R0) + RET + +empty_seqs: + // Return value + MOVD $0x01, R16 + MOVB R16, ret+8(FP) + RET + +// func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: CMOV, SSE +TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD $0, R1 + MOVD R1, 16(RSP) + MOVD R1, 24(RSP) + MOVD R1, 32(RSP) + MOVD 112(R0), R9 + MOVD 128(R0), R1 + MOVD R1, 40(RSP) + MOVD 144(R0), R10 + MOVD 136(R0), R11 + MOVD 200(R0), R1 + MOVD R1, 64(RSP) + MOVD 176(R0), R1 + MOVD R1, 56(RSP) + MOVD 184(R0), R0 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 + ADD R0, R16, R16 + MOVD R16, 56(RSP) + + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) + MOVD 40(RSP), R16 + ADD R9, R16, R16 + MOVD R16, 40(RSP) + + // outBase += outPosition + ADD R11, R9, R9 + +sequenceDecs_decodeSync_amd64_main_loop: + MOVD 8(RSP), R12 + + // Fill bitreader to have enough for the offset and match length. + CMP $0x08, R5 + BLT sequenceDecs_decodeSync_amd64_fill_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R12, R12 + MOVD (R12), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decodeSync_amd64_fill_end + +sequenceDecs_decodeSync_amd64_fill_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decodeSync_amd64_fill_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decodeSync_amd64_fill_end + LSL $0x08, R2, R2 + SUB $0x01, R12, R12 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R12), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decodeSync_amd64_fill_byte_by_byte + +sequenceDecs_decodeSync_amd64_fill_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decodeSync_amd64_fill_end: + // Update offset + MOVD R8, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_amd64_of_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_amd64_of_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_amd64_of_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_amd64_of_update_zero: + MOVD R0, 16(RSP) + + // Update match length + MOVD R7, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_amd64_ml_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_amd64_ml_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_amd64_ml_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_amd64_ml_update_zero: + MOVD R0, 24(RSP) + + // Fill bitreader to have enough for the remaining + CMP $0x08, R5 + BLT sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R12, R12 + MOVD (R12), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decodeSync_amd64_fill_2_end + +sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decodeSync_amd64_fill_2_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decodeSync_amd64_fill_2_end + LSL $0x08, R2, R2 + SUB $0x01, R12, R12 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R12), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte + +sequenceDecs_decodeSync_amd64_fill_2_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decodeSync_amd64_fill_2_end: + // Update literal length + MOVD R6, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_amd64_ll_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_amd64_ll_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_amd64_ll_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_amd64_ll_update_zero: + MOVD R0, 32(RSP) + + // Fill bitreader for state updates + MOVD R12, 8(RSP) + MOVD R8, R0 + LSR $0x08, R0, R0 + MOVBU R0, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_amd64_skip_update + + // Update Literal Length State + MOVBU R6, R12 + LSRW $0x10, R6, R6 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R6, R6 + + // Load ctx.llTable + MOVD ctx+16(FP), R1 + MOVD (R1), R1 + ADD R6<<3, R1, R15 + MOVD (R15), R6 + + // Update Match Length State + MOVBU R7, R12 + LSRW $0x10, R7, R7 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R7, R7 + + // Load ctx.mlTable + MOVD ctx+16(FP), R1 + MOVD 24(R1), R1 + ADD R7<<3, R1, R15 + MOVD (R15), R7 + + // Update Offset State + MOVBU R8, R12 + LSRW $0x10, R8, R8 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R8, R8 + + // Load ctx.ofTable + MOVD ctx+16(FP), R1 + MOVD 48(R1), R1 + ADD R8<<3, R1, R15 + MOVD (R15), R8 + +sequenceDecs_decodeSync_amd64_skip_update: + // Adjust offset + MOVD s+0(FP), R1 + MOVD 16(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 + ADD $144, R1, R15 + VLD1 (R15), [V0.B16] + MOVD R12, 144(R1) + ADD $152, R1, R15 + VST1 [V0.B16], (R15) + JMP sequenceDecs_decodeSync_amd64_after_adjust + +sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: + MOVD 32(RSP), R16 + CMP $0x00000000, R16 + BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero + ADD $1, R12, R12 + JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero + +sequenceDecs_decodeSync_amd64_adjust_offset_maybezero: + TST R12, R12 + BNE sequenceDecs_decodeSync_amd64_adjust_offset_nonzero + MOVD 144(R1), R12 + JMP sequenceDecs_decodeSync_amd64_after_adjust + +sequenceDecs_decodeSync_amd64_adjust_offset_nonzero: + MOVD R12, R0 + MOVD $0, R13 + MOVD $-1, R14 + CMP $0x03, R12 + CSEL EQ, R13, R0, R0 + CSEL EQ, R14, R13, R13 + ADD R0<<3, R1, R15 + MOVD 144(R15), R16 + ADDS R16, R13, R13 + BNE sequenceDecs_decodeSync_amd64_adjust_temp_valid + MOVD $0x00000001, R13 + +sequenceDecs_decodeSync_amd64_adjust_temp_valid: + CMP $0x01, R12 + BEQ sequenceDecs_decodeSync_amd64_adjust_skip + MOVD 152(R1), R0 + MOVD R0, 160(R1) + +sequenceDecs_decodeSync_amd64_adjust_skip: + MOVD 144(R1), R0 + MOVD R0, 152(R1) + MOVD R13, 144(R1) + MOVD R13, R12 + +sequenceDecs_decodeSync_amd64_after_adjust: + MOVD R12, 16(RSP) + + // Check values + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 + ADD R1, R0, R13 + MOVD s+0(FP), R14 + MOVD 256(R14), R16 + ADD R13, R16, R16 + MOVD R16, 256(R14) + MOVD ctx+16(FP), R13 + MOVD 104(R13), R16 + SUBS R1, R16, R16 + MOVD R16, 104(R13) + BMI error_not_enough_literals + CMP $0x00020002, R0 + BHI sequenceDecs_decodeSync_amd64_error_match_len_too_big + TST R12, R12 + BNE sequenceDecs_decodeSync_amd64_match_len_ofs_ok + TST R0, R0 + BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_amd64_match_len_ofs_ok: + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 + + // Check if we have enough space in s.out + ADD R12, R0, R13 + ADD $16, R13, R13 + ADD R9, R13, R13 + MOVD 40(RSP), R16 + CMP R16, R13 + BHI error_not_enough_space + + // Copy literals + TST R0, R0 + BEQ check_offset + MOVD $0, R13 + +copy_1: + ADD R13, R10, R15 + VLD1 (R15), [V0.B16] + ADD R13, R9, R15 + VST1 [V0.B16], (R15) + ADD $0x10, R13, R13 + CMP R0, R13 + BLO copy_1 + ADD R0, R10, R10 + ADD R0, R9, R9 + ADD R0, R11, R11 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVD R11, R0 + MOVD 48(RSP), R16 + ADD R16, R0, R0 + CMP R0, R1 + BGT error_match_off_too_big + MOVD 64(RSP), R16 + CMP R16, R1 + BGT error_match_off_too_big + + // Copy match from history + MOVD R1, R0 + SUBS R11, R0, R0 + BLS copy_match + MOVD 56(RSP), R13 + SUB R0, R13, R13 + CMP R0, R12 + BGT copy_all_from_history + MOVD R12, R0 + SUBS $0x10, R0, R0 + BLO copy_4_small + +copy_4_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R0, R0 + BHS copy_4_loop + ADD R0, R13, R13 + ADD $16, R13, R13 + ADD R0, R9, R9 + ADD $16, R9, R9 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_4_end + +copy_4_small: + CMP $0x03, R12 + BEQ copy_4_move_3 + CMP $0x08, R12 + BLO copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end + +copy_4_move_4through7: + MOVWU (R13), R0 + ADD R12, R13, R15 + MOVWU -4(R15), R1 + MOVW R0, (R9) + ADD R12, R9, R15 + MOVW R1, -4(R15) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end + +copy_4_move_8through16: + MOVD (R13), R0 + ADD R12, R13, R15 + MOVD -8(R15), R1 + MOVD R0, (R9) + ADD R12, R9, R15 + MOVD R1, -8(R15) + ADD R12, R13, R13 + ADD R12, R9, R9 + +copy_4_end: + ADD R12, R11, R11 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVD R0, R14 + SUBS $0x10, R14, R14 + BLO copy_5_small + +copy_5_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_5_end + +copy_5_small: + CMP $0x03, R0 + BEQ copy_5_move_3 + BLO copy_5_move_1or2 + CMP $0x08, R0 + BLO copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_3: + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_4through7: + MOVWU (R13), R14 + ADD R0, R13, R15 + MOVWU -4(R15), R4 + MOVW R14, (R9) + ADD R0, R9, R15 + MOVW R4, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_8through16: + MOVD (R13), R14 + ADD R0, R13, R15 + MOVD -8(R15), R4 + MOVD R14, (R9) + ADD R0, R9, R15 + MOVD R4, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + +copy_5_end: + ADD R0, R11, R11 + SUB R0, R12, R12 + + // Copy match from the current buffer +copy_match: + MOVD R9, R0 + SUB R1, R0, R0 + + // ml <= mo + CMP R1, R12 + BHI copy_overlapping_match + + // Copy non-overlapping match + ADD R12, R11, R11 + MOVD R9, R1 + ADD R12, R9, R9 + +copy_2: + VLD1 (R0), [V0.B16] + VST1 [V0.B16], (R1) + ADD $0x10, R0, R0 + ADD $0x10, R1, R1 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADD R12, R11, R11 + +copy_slow_3: + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 + +handle_loop: + MOVD ctx+16(FP), R0 + MOVD 96(R0), R16 + SUBS $1, R16, R16 + MOVD R16, 96(R0) + BPL sequenceDecs_decodeSync_amd64_main_loop + +loop_finished: + MOVD br+8(FP), R0 + MOVD R2, 24(R0) + MOVB R3, 40(R0) + MOVD R5, 32(R0) + + // Update the context + MOVD ctx+16(FP), R0 + MOVD R11, 136(R0) + MOVD 144(R0), R1 + SUB R1, R10, R10 + MOVD R10, 168(R0) + + // Return success + MOVD $0x00000000, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: + MOVD 24(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 216(R1) + MOVD $0x00000001, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_amd64_error_match_len_too_big: + MOVD ctx+16(FP), R0 + MOVD 24(RSP), R1 + MOVD R1, 216(R0) + MOVD $0x00000002, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVD ctx+16(FP), R0 + MOVD 16(RSP), R1 + MOVD R1, 224(R0) + MOVD R11, 136(R0) + MOVD $0x00000003, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVD ctx+16(FP), R0 + MOVD 32(RSP), R1 + MOVD R1, 208(R0) + MOVD $0x00000004, R16 + MOVD R16, ret+24(FP) + RET + + // Return with overread error +error_overread: + MOVD $0x00000006, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVD ctx+16(FP), R0 + MOVD 32(RSP), R1 + MOVD R1, 208(R0) + MOVD 24(RSP), R1 + MOVD R1, 216(R0) + MOVD R11, 136(R0) + MOVD $0x00000005, R16 + MOVD R16, ret+24(FP) + RET + +// skipped sequenceDecs_decodeSync_bmi2 (generic twin preferred on arm64) + +// func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: CMOV, SSE +TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD $0, R1 + MOVD R1, 16(RSP) + MOVD R1, 24(RSP) + MOVD R1, 32(RSP) + MOVD 112(R0), R9 + MOVD 128(R0), R1 + MOVD R1, 40(RSP) + MOVD 144(R0), R10 + MOVD 136(R0), R11 + MOVD 200(R0), R1 + MOVD R1, 64(RSP) + MOVD 176(R0), R1 + MOVD R1, 56(RSP) + MOVD 184(R0), R0 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 + ADD R0, R16, R16 + MOVD R16, 56(RSP) + + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) + MOVD 40(RSP), R16 + ADD R9, R16, R16 + MOVD R16, 40(RSP) + + // outBase += outPosition + ADD R11, R9, R9 + +sequenceDecs_decodeSync_safe_amd64_main_loop: + MOVD 8(RSP), R12 + + // Fill bitreader to have enough for the offset and match length. + CMP $0x08, R5 + BLT sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R12, R12 + MOVD (R12), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decodeSync_safe_amd64_fill_end + +sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decodeSync_safe_amd64_fill_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decodeSync_safe_amd64_fill_end + LSL $0x08, R2, R2 + SUB $0x01, R12, R12 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R12), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte + +sequenceDecs_decodeSync_safe_amd64_fill_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decodeSync_safe_amd64_fill_end: + // Update offset + MOVD R8, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_safe_amd64_of_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_safe_amd64_of_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_safe_amd64_of_update_zero: + MOVD R0, 16(RSP) + + // Update match length + MOVD R7, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_safe_amd64_ml_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_safe_amd64_ml_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_safe_amd64_ml_update_zero: + MOVD R0, 24(RSP) + + // Fill bitreader to have enough for the remaining + CMP $0x08, R5 + BLT sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte + MOVD R3, R0 + LSR $0x03, R0, R0 + SUB R0, R12, R12 + MOVD (R12), R2 + SUB R0, R5, R5 + AND $0x07, R3, R3 + JMP sequenceDecs_decodeSync_safe_amd64_fill_2_end + +sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte: + CMP $0x00, R5 + BLE sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread + CMP $0x07, R3 + BLE sequenceDecs_decodeSync_safe_amd64_fill_2_end + LSL $0x08, R2, R2 + SUB $0x01, R12, R12 + SUB $0x01, R5, R5 + SUB $0x08, R3, R3 + MOVBU (R12), R0 + ORR R0, R2, R2 + JMP sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte + +sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: + CMP $0x40, R3 + BHI error_overread + +sequenceDecs_decodeSync_safe_amd64_fill_2_end: + // Update literal length + MOVD R6, R0 + MOVD R3, R1 + MOVD R2, R13 + LSL R1, R13, R13 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 + LSR $0x20, R0, R0 + TST R1, R1 + BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero + ADD R1, R3, R3 + CMP $0x40, R3 + BHI sequenceDecs_decodeSync_safe_amd64_ll_update_zero + CMP $0x40, R1 + BHS sequenceDecs_decodeSync_safe_amd64_ll_update_zero + NEG R1, R1 + LSR R1, R13, R13 + ADD R13, R0, R0 + +sequenceDecs_decodeSync_safe_amd64_ll_update_zero: + MOVD R0, 32(RSP) + + // Fill bitreader for state updates + MOVD R12, 8(RSP) + MOVD R8, R0 + LSR $0x08, R0, R0 + MOVBU R0, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_safe_amd64_skip_update + + // Update Literal Length State + MOVBU R6, R12 + LSRW $0x10, R6, R6 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R6, R6 + + // Load ctx.llTable + MOVD ctx+16(FP), R1 + MOVD (R1), R1 + ADD R6<<3, R1, R15 + MOVD (R15), R6 + + // Update Match Length State + MOVBU R7, R12 + LSRW $0x10, R7, R7 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R7, R7 + + // Load ctx.mlTable + MOVD ctx+16(FP), R1 + MOVD 24(R1), R1 + ADD R7<<3, R1, R15 + MOVD (R15), R7 + + // Update Offset State + MOVBU R8, R12 + LSRW $0x10, R8, R8 + ADD R12, R3, R1 + MOVD R2, R13 + MOVD R1, R3 + NEG R1, R16 + ROR R16, R13, R13 + MOVD $0x00000001, R14 + BFI $0, R12, $8, R1 + LSLW R1, R14, R14 + SUBW $1, R14, R14 + AND R14, R13, R13 + ADD R13, R8, R8 + + // Load ctx.ofTable + MOVD ctx+16(FP), R1 + MOVD 48(R1), R1 + ADD R8<<3, R1, R15 + MOVD (R15), R8 + +sequenceDecs_decodeSync_safe_amd64_skip_update: + // Adjust offset + MOVD s+0(FP), R1 + MOVD 16(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 + ADD $144, R1, R15 + VLD1 (R15), [V0.B16] + MOVD R12, 144(R1) + ADD $152, R1, R15 + VST1 [V0.B16], (R15) + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + +sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: + MOVD 32(RSP), R16 + CMP $0x00000000, R16 + BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero + ADD $1, R12, R12 + JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero + +sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero: + TST R12, R12 + BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero + MOVD 144(R1), R12 + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + +sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero: + MOVD R12, R0 + MOVD $0, R13 + MOVD $-1, R14 + CMP $0x03, R12 + CSEL EQ, R13, R0, R0 + CSEL EQ, R14, R13, R13 + ADD R0<<3, R1, R15 + MOVD 144(R15), R16 + ADDS R16, R13, R13 + BNE sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid + MOVD $0x00000001, R13 + +sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid: + CMP $0x01, R12 + BEQ sequenceDecs_decodeSync_safe_amd64_adjust_skip + MOVD 152(R1), R0 + MOVD R0, 160(R1) + +sequenceDecs_decodeSync_safe_amd64_adjust_skip: + MOVD 144(R1), R0 + MOVD R0, 152(R1) + MOVD R13, 144(R1) + MOVD R13, R12 + +sequenceDecs_decodeSync_safe_amd64_after_adjust: + MOVD R12, 16(RSP) + + // Check values + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 + ADD R1, R0, R13 + MOVD s+0(FP), R14 + MOVD 256(R14), R16 + ADD R13, R16, R16 + MOVD R16, 256(R14) + MOVD ctx+16(FP), R13 + MOVD 104(R13), R16 + SUBS R1, R16, R16 + MOVD R16, 104(R13) + BMI error_not_enough_literals + CMP $0x00020002, R0 + BHI sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big + TST R12, R12 + BNE sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok + TST R0, R0 + BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 + + // Check if we have enough space in s.out + ADD R12, R0, R13 + ADD R9, R13, R13 + MOVD 40(RSP), R16 + CMP R16, R13 + BHI error_not_enough_space + + // Copy literals + TST R0, R0 + BEQ check_offset + MOVD R0, R13 + SUBS $0x10, R13, R13 + BLO copy_1_small + +copy_1_loop: + VLD1 (R10), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R10, R10 + ADD $0x10, R9, R9 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R10, R10 + ADD $16, R10, R10 + ADD R13, R9, R9 + ADD $16, R9, R9 + ADD $-16, R10, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_1_end + +copy_1_small: + CMP $0x03, R0 + BEQ copy_1_move_3 + BLO copy_1_move_1or2 + CMP $0x08, R0 + BLO copy_1_move_4through7 + JMP copy_1_move_8through16 + +copy_1_move_1or2: + MOVBU (R10), R16 + BFI $0, R16, $8, R13 + ADD R0, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R9) + ADD R0, R9, R15 + MOVB R14, -1(R15) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end + +copy_1_move_3: + MOVHU (R10), R13 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R9) + MOVB R14, 2(R9) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end + +copy_1_move_4through7: + MOVWU (R10), R13 + ADD R0, R10, R15 + MOVWU -4(R15), R14 + MOVW R13, (R9) + ADD R0, R9, R15 + MOVW R14, -4(R15) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end + +copy_1_move_8through16: + MOVD (R10), R13 + ADD R0, R10, R15 + MOVD -8(R15), R14 + MOVD R13, (R9) + ADD R0, R9, R15 + MOVD R14, -8(R15) + ADD R0, R10, R10 + ADD R0, R9, R9 + +copy_1_end: + ADD R0, R11, R11 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVD R11, R0 + MOVD 48(RSP), R16 + ADD R16, R0, R0 + CMP R0, R1 + BGT error_match_off_too_big + MOVD 64(RSP), R16 + CMP R16, R1 + BGT error_match_off_too_big + + // Copy match from history + MOVD R1, R0 + SUBS R11, R0, R0 + BLS copy_match + MOVD 56(RSP), R13 + SUB R0, R13, R13 + CMP R0, R12 + BGT copy_all_from_history + MOVD R12, R0 + SUBS $0x10, R0, R0 + BLO copy_4_small + +copy_4_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R0, R0 + BHS copy_4_loop + ADD R0, R13, R13 + ADD $16, R13, R13 + ADD R0, R9, R9 + ADD $16, R9, R9 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_4_end + +copy_4_small: + CMP $0x03, R12 + BEQ copy_4_move_3 + CMP $0x08, R12 + BLO copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end + +copy_4_move_4through7: + MOVWU (R13), R0 + ADD R12, R13, R15 + MOVWU -4(R15), R1 + MOVW R0, (R9) + ADD R12, R9, R15 + MOVW R1, -4(R15) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end + +copy_4_move_8through16: + MOVD (R13), R0 + ADD R12, R13, R15 + MOVD -8(R15), R1 + MOVD R0, (R9) + ADD R12, R9, R15 + MOVD R1, -8(R15) + ADD R12, R13, R13 + ADD R12, R9, R9 + +copy_4_end: + ADD R12, R11, R11 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVD R0, R14 + SUBS $0x10, R14, R14 + BLO copy_5_small + +copy_5_loop: + VLD1 (R13), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + ADD $-16, R13, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_5_end + +copy_5_small: + CMP $0x03, R0 + BEQ copy_5_move_3 + BLO copy_5_move_1or2 + CMP $0x08, R0 + BLO copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_3: + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_4through7: + MOVWU (R13), R14 + ADD R0, R13, R15 + MOVWU -4(R15), R4 + MOVW R14, (R9) + ADD R0, R9, R15 + MOVW R4, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end + +copy_5_move_8through16: + MOVD (R13), R14 + ADD R0, R13, R15 + MOVD -8(R15), R4 + MOVD R14, (R9) + ADD R0, R9, R15 + MOVD R4, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + +copy_5_end: + ADD R0, R11, R11 + SUB R0, R12, R12 + + // Copy match from the current buffer +copy_match: + MOVD R9, R0 + SUB R1, R0, R0 + + // ml <= mo + CMP R1, R12 + BHI copy_overlapping_match + + // Copy non-overlapping match + ADD R12, R11, R11 + MOVD R12, R1 + SUBS $0x10, R1, R1 + BLO copy_2_small + +copy_2_loop: + VLD1 (R0), [V0.B16] + VST1 [V0.B16], (R9) + ADD $0x10, R0, R0 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_2_loop + ADD R1, R0, R0 + ADD $16, R0, R0 + ADD R1, R9, R9 + ADD $16, R9, R9 + ADD $-16, R0, R15 + VLD1 (R15), [V0.B16] + ADD $-16, R9, R15 + VST1 [V0.B16], (R15) + JMP copy_2_end + +copy_2_small: + CMP $0x03, R12 + BEQ copy_2_move_3 + BLO copy_2_move_1or2 + CMP $0x08, R12 + BLO copy_2_move_4through7 + JMP copy_2_move_8through16 + +copy_2_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + ADD R12, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R1, (R9) + ADD R12, R9, R15 + MOVB R13, -1(R15) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end + +copy_2_move_3: + MOVHU (R0), R1 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R13 + MOVH R1, (R9) + MOVB R13, 2(R9) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end + +copy_2_move_4through7: + MOVWU (R0), R1 + ADD R12, R0, R15 + MOVWU -4(R15), R13 + MOVW R1, (R9) + ADD R12, R9, R15 + MOVW R13, -4(R15) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end + +copy_2_move_8through16: + MOVD (R0), R1 + ADD R12, R0, R15 + MOVD -8(R15), R13 + MOVD R1, (R9) + ADD R12, R9, R15 + MOVD R13, -8(R15) + ADD R12, R0, R0 + ADD R12, R9, R9 + +copy_2_end: + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADD R12, R11, R11 + +copy_slow_3: + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 + +handle_loop: + MOVD ctx+16(FP), R0 + MOVD 96(R0), R16 + SUBS $1, R16, R16 + MOVD R16, 96(R0) + BPL sequenceDecs_decodeSync_safe_amd64_main_loop + +loop_finished: + MOVD br+8(FP), R0 + MOVD R2, 24(R0) + MOVB R3, 40(R0) + MOVD R5, 32(R0) + + // Update the context + MOVD ctx+16(FP), R0 + MOVD R11, 136(R0) + MOVD 144(R0), R1 + SUB R1, R10, R10 + MOVD R10, 168(R0) + + // Return success + MOVD $0x00000000, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: + MOVD 24(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 216(R1) + MOVD $0x00000001, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: + MOVD ctx+16(FP), R0 + MOVD 24(RSP), R1 + MOVD R1, 216(R0) + MOVD $0x00000002, R16 + MOVD R16, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVD ctx+16(FP), R0 + MOVD 16(RSP), R1 + MOVD R1, 224(R0) + MOVD R11, 136(R0) + MOVD $0x00000003, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVD ctx+16(FP), R0 + MOVD 32(RSP), R1 + MOVD R1, 208(R0) + MOVD $0x00000004, R16 + MOVD R16, ret+24(FP) + RET + + // Return with overread error +error_overread: + MOVD $0x00000006, R16 + MOVD R16, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVD ctx+16(FP), R0 + MOVD 32(RSP), R1 + MOVD R1, 208(R0) + MOVD 24(RSP), R1 + MOVD R1, 216(R0) + MOVD R11, 136(R0) + MOVD $0x00000005, R16 + MOVD R16, ret+24(FP) + RET + +// skipped sequenceDecs_decodeSync_safe_bmi2 (generic twin preferred on arm64) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go new file mode 100644 index 000000000..42ce5de71 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -0,0 +1,316 @@ +//go:build (amd64 || arm64) && !appengine && !noasm && gc + +package zstd + +import ( + "fmt" + "io" +) + +// This file holds the parts of the assembly sequence decoder that are identical +// across architectures: the context structs exchanged with the asm, the error +// codes, and the decode/decodeSync/executeSimple wrappers. Each architecture +// supplies the small dispatch helpers (decodeAsm, decodeSyncAsm, +// executeSimpleAsm) that select the concrete asm routine — amd64 also chooses a +// BMI2 variant, arm64 has a single implementation. + +type decodeSyncAsmContext struct { + llTable []decSymbol + mlTable []decSymbol + ofTable []decSymbol + llState uint64 + mlState uint64 + ofState uint64 + iteration int + litRemain int + out []byte + outPosition int + literals []byte + litPosition int + history []byte + windowSize int + ll int // set on error (not for all errors, please refer to _generate/gen.go) + ml int // set on error (not for all errors, please refer to _generate/gen.go) + mo int // set on error (not for all errors, please refer to _generate/gen.go) +} + +type decodeAsmContext struct { + llTable []decSymbol + mlTable []decSymbol + ofTable []decSymbol + llState uint64 + mlState uint64 + ofState uint64 + iteration int + seqs []seqVals + litRemain int +} + +type executeAsmContext struct { + seqs []seqVals + seqIndex int + out []byte + history []byte + literals []byte + outPosition int + litPosition int + windowSize int +} + +const noError = 0 + +// error reported when mo == 0 && ml > 0 +const errorMatchLenOfsMismatch = 1 + +// error reported when ml > maxMatchLen +const errorMatchLenTooBig = 2 + +// error reported when mo > available history or mo > s.windowSize +const errorMatchOffTooBig = 3 + +// error reported when the sum of literal lengths exeeceds the literal buffer size +const errorNotEnoughLiterals = 4 + +// error reported when capacity of `out` is too small +const errorNotEnoughSpace = 5 + +// error reported when bits are overread. +const errorOverread = 6 + +// useSafeDecodeSync reports whether decodeSyncSimple must use the bounds-exact +// ("safe") copy variants for the current buffer geometry. +// +// When the output and literal buffers have compressedBlockOverAlloc (16) +// bytes of slack past their logical use, the assembly may use extended +// memory copies that read and write in 16-byte blocks, overrunning the end +// of a literal run or match by up to 15 bytes. Otherwise it must use the +// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic +// selection in executeSimple below. +// +// The unsafe copies were disabled in #644 (2022) as a mitigation for a +// crash, but that crash's root cause — an unguarded bitReader overread that +// produced out-of-range match offsets/lengths — was fixed three days later +// in #645, which also added the fuzz corpus that has guarded this path since. +// See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml) +// covers the extended-copy path, which -race and plain fuzzing cannot. +func (s *sequenceDecs) useSafeDecodeSync() bool { + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + return true + } + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + return true + } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + return true + } + return false +} + +// decode sequences from the stream with the provided history but without a dictionary. +func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { + if len(s.dict) > 0 { + return false, nil + } + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { + return false, nil + } + + useSafe := s.useSafeDecodeSync() + + br := s.br + + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) + + ctx := decodeSyncAsmContext{ + llTable: s.litLengths.fse.dt[:maxTablesize], + mlTable: s.matchLengths.fse.dt[:maxTablesize], + ofTable: s.offsets.fse.dt[:maxTablesize], + llState: uint64(s.litLengths.state.state), + mlState: uint64(s.matchLengths.state.state), + ofState: uint64(s.offsets.state.state), + iteration: s.nSeqs - 1, + litRemain: len(s.literals), + out: s.out, + outPosition: len(s.out), + literals: s.literals, + windowSize: s.windowSize, + history: hist, + } + + s.seqSize = 0 + startSize := len(s.out) + + errCode := decodeSyncAsm(s, br, &ctx, useSafe) + switch errCode { + case noError: + break + + case errorMatchLenOfsMismatch: + return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml) + + case errorMatchLenTooBig: + return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml) + + case errorMatchOffTooBig: + return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", + ctx.mo, ctx.outPosition+len(hist)-startSize) + + case errorNotEnoughLiterals: + return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", + ctx.ll, ctx.litRemain+ctx.ll) + + case errorOverread: + return true, io.ErrUnexpectedEOF + + case errorNotEnoughSpace: + size := ctx.outPosition + ctx.ll + ctx.ml + if debugDecoder { + println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize) + } + return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + + default: + return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) + } + + s.seqSize += ctx.litRemain + if s.seqSize > maxBlockSize { + return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } + err := br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + return true, err + } + + s.literals = s.literals[ctx.litPosition:] + t := ctx.outPosition + s.out = s.out[:t] + + // Add final literals + s.out = append(s.out, s.literals...) + if debugDecoder { + t += len(s.literals) + if t != len(s.out) { + panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t)) + } + } + + return true, nil +} + +// decode sequences from the stream without the provided history. +func (s *sequenceDecs) decode(seqs []seqVals) error { + br := s.br + + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) + + ctx := decodeAsmContext{ + llTable: s.litLengths.fse.dt[:maxTablesize], + mlTable: s.matchLengths.fse.dt[:maxTablesize], + ofTable: s.offsets.fse.dt[:maxTablesize], + llState: uint64(s.litLengths.state.state), + mlState: uint64(s.matchLengths.state.state), + ofState: uint64(s.offsets.state.state), + seqs: seqs, + iteration: len(seqs) - 1, + litRemain: len(s.literals), + } + + if debugDecoder { + println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream") + } + + s.seqSize = 0 + lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 + errCode := decodeAsm(s, br, &ctx, lte56bits) + if errCode != 0 { + i := len(seqs) - ctx.iteration - 1 + switch errCode { + case errorMatchLenOfsMismatch: + ml := ctx.seqs[i].ml + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + + case errorMatchLenTooBig: + ml := ctx.seqs[i].ml + return fmt.Errorf("match len (%d) bigger than max allowed length", ml) + + case errorNotEnoughLiterals: + ll := ctx.seqs[i].ll + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) + case errorOverread: + return io.ErrUnexpectedEOF + } + + return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) + } + + if ctx.litRemain < 0 { + return fmt.Errorf("literal count is too big: total available %d, total requested %d", + len(s.literals), len(s.literals)-ctx.litRemain) + } + + s.seqSize += ctx.litRemain + if s.seqSize > maxBlockSize { + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } + if debugDecoder { + println("decode: ", br.remain(), "bits remain on stream. code:", errCode) + } + err := br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + } + return err +} + +// executeSimple handles cases when dictionary is not used. +func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { + // Ensure we have enough output size... + if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { + addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] + } + + if debugDecoder { + printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) + } + + var t = len(s.out) + out := s.out[:t+s.seqSize] + + ctx := executeAsmContext{ + seqs: seqs, + seqIndex: 0, + out: out, + history: hist, + outPosition: t, + litPosition: 0, + literals: s.literals, + windowSize: s.windowSize, + } + // useSafe avoids overwriting the output buffer when the literals slice has + // not been allocated with the required over-allocation slack. + useSafe := cap(s.literals) < len(s.literals)+compressedBlockOverAlloc + + ok := executeSimpleAsm(&ctx, useSafe) + if !ok { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", + seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) + } + s.literals = s.literals[ctx.litPosition:] + t = ctx.outPosition + + // Add final literals + copy(out[t:], s.literals) + if debugDecoder { + t += len(s.literals) + if t != len(out) { + panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) + } + } + s.out = out + + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go index 516cd9b07..8a3db6ba2 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index 336c28893..36c56f36d 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -334,9 +334,10 @@ func decodeSnappy(blk *blockEnc, src []byte) error { return errUnsupportedLiteralLength } - //if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s { - // return ErrSnappyCorrupt - //} + if length > len(src)-s { + println("length > len(src)-s", length, len(src)-s) + return ErrSnappyCorrupt + } blk.literals = append(blk.literals, src[s:s+length]...) //println(length, "litLen") diff --git a/vendor/github.com/mitchellh/go-homedir/LICENSE b/vendor/github.com/mitchellh/go-homedir/LICENSE deleted file mode 100644 index f9c841a51..000000000 --- a/vendor/github.com/mitchellh/go-homedir/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/go-homedir/README.md b/vendor/github.com/mitchellh/go-homedir/README.md deleted file mode 100644 index d70706d5b..000000000 --- a/vendor/github.com/mitchellh/go-homedir/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# go-homedir - -This is a Go library for detecting the user's home directory without -the use of cgo, so the library can be used in cross-compilation environments. - -Usage is incredibly simple, just call `homedir.Dir()` to get the home directory -for a user, and `homedir.Expand()` to expand the `~` in a path to the home -directory. - -**Why not just use `os/user`?** The built-in `os/user` package requires -cgo on Darwin systems. This means that any Go code that uses that package -cannot cross compile. But 99% of the time the use for `os/user` is just to -retrieve the home directory, which we can do for the current user without -cgo. This library does that, enabling cross-compilation. diff --git a/vendor/github.com/mitchellh/go-homedir/homedir.go b/vendor/github.com/mitchellh/go-homedir/homedir.go deleted file mode 100644 index 25378537e..000000000 --- a/vendor/github.com/mitchellh/go-homedir/homedir.go +++ /dev/null @@ -1,167 +0,0 @@ -package homedir - -import ( - "bytes" - "errors" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" -) - -// DisableCache will disable caching of the home directory. Caching is enabled -// by default. -var DisableCache bool - -var homedirCache string -var cacheLock sync.RWMutex - -// Dir returns the home directory for the executing user. -// -// This uses an OS-specific method for discovering the home directory. -// An error is returned if a home directory cannot be detected. -func Dir() (string, error) { - if !DisableCache { - cacheLock.RLock() - cached := homedirCache - cacheLock.RUnlock() - if cached != "" { - return cached, nil - } - } - - cacheLock.Lock() - defer cacheLock.Unlock() - - var result string - var err error - if runtime.GOOS == "windows" { - result, err = dirWindows() - } else { - // Unix-like system, so just assume Unix - result, err = dirUnix() - } - - if err != nil { - return "", err - } - homedirCache = result - return result, nil -} - -// Expand expands the path to include the home directory if the path -// is prefixed with `~`. If it isn't prefixed with `~`, the path is -// returned as-is. -func Expand(path string) (string, error) { - if len(path) == 0 { - return path, nil - } - - if path[0] != '~' { - return path, nil - } - - if len(path) > 1 && path[1] != '/' && path[1] != '\\' { - return "", errors.New("cannot expand user-specific home dir") - } - - dir, err := Dir() - if err != nil { - return "", err - } - - return filepath.Join(dir, path[1:]), nil -} - -// Reset clears the cache, forcing the next call to Dir to re-detect -// the home directory. This generally never has to be called, but can be -// useful in tests if you're modifying the home directory via the HOME -// env var or something. -func Reset() { - cacheLock.Lock() - defer cacheLock.Unlock() - homedirCache = "" -} - -func dirUnix() (string, error) { - homeEnv := "HOME" - if runtime.GOOS == "plan9" { - // On plan9, env vars are lowercase. - homeEnv = "home" - } - - // First prefer the HOME environmental variable - if home := os.Getenv(homeEnv); home != "" { - return home, nil - } - - var stdout bytes.Buffer - - // If that fails, try OS specific commands - if runtime.GOOS == "darwin" { - cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`) - cmd.Stdout = &stdout - if err := cmd.Run(); err == nil { - result := strings.TrimSpace(stdout.String()) - if result != "" { - return result, nil - } - } - } else { - cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - // If the error is ErrNotFound, we ignore it. Otherwise, return it. - if err != exec.ErrNotFound { - return "", err - } - } else { - if passwd := strings.TrimSpace(stdout.String()); passwd != "" { - // username:password:uid:gid:gecos:home:shell - passwdParts := strings.SplitN(passwd, ":", 7) - if len(passwdParts) > 5 { - return passwdParts[5], nil - } - } - } - } - - // If all else fails, try the shell - stdout.Reset() - cmd := exec.Command("sh", "-c", "cd && pwd") - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - return "", err - } - - result := strings.TrimSpace(stdout.String()) - if result == "" { - return "", errors.New("blank output when reading home directory") - } - - return result, nil -} - -func dirWindows() (string, error) { - // First prefer the HOME environmental variable - if home := os.Getenv("HOME"); home != "" { - return home, nil - } - - // Prefer standard environment variable USERPROFILE - if home := os.Getenv("USERPROFILE"); home != "" { - return home, nil - } - - drive := os.Getenv("HOMEDRIVE") - path := os.Getenv("HOMEPATH") - home := drive + path - if drive == "" || path == "" { - return "", errors.New("HOMEDRIVE, HOMEPATH, or USERPROFILE are blank") - } - - return home, nil -} diff --git a/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count b/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count deleted file mode 100644 index dfd78a88d..000000000 Binary files a/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/asm@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count and /dev/null differ diff --git a/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/weekends b/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/weekends deleted file mode 100644 index 1e8b31496..000000000 --- a/vendor/github.com/modern-go/reflect2/~/Library/Application Support/go/telemetry/local/weekends +++ /dev/null @@ -1 +0,0 @@ -6 diff --git a/vendor/github.com/onsi/gomega/CHANGELOG.md b/vendor/github.com/onsi/gomega/CHANGELOG.md index e7601cfde..8f1d34e38 100644 --- a/vendor/github.com/onsi/gomega/CHANGELOG.md +++ b/vendor/github.com/onsi/gomega/CHANGELOG.md @@ -1,3 +1,13 @@ +## 1.43.0 + +### Features + +Add gomock adaptor extension for using Gomega matchers with gomock + +## 1.42.1 + +Bump Dependencies + ## 1.42.0 Add a set of Claude skill as a marketplace plugin diff --git a/vendor/github.com/onsi/gomega/gomega_dsl.go b/vendor/github.com/onsi/gomega/gomega_dsl.go index 967d90d0d..3206e05da 100644 --- a/vendor/github.com/onsi/gomega/gomega_dsl.go +++ b/vendor/github.com/onsi/gomega/gomega_dsl.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega/types" ) -const GOMEGA_VERSION = "1.42.0" +const GOMEGA_VERSION = "1.43.0" const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler. If you're using Ginkgo then you probably forgot to put your assertion in an It(). diff --git a/vendor/github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header/header.go b/vendor/github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header/header.go index 8547c8dfd..820bf436a 100644 --- a/vendor/github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header/header.go +++ b/vendor/github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header/header.go @@ -90,7 +90,7 @@ loop: s = skipSpace(s[1:]) } } - return + return specs } func skipSpace(s string) (rest string) { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/counter.go b/vendor/github.com/prometheus/client_golang/prometheus/counter.go index 4ce84e7a8..7d963d3af 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/counter.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/counter.go @@ -85,11 +85,12 @@ type CounterVecOpts struct { // Both internal tracking values are added up in the Write method. This has to // be taken into account when it comes to precision and overflow behavior. func NewCounter(opts CounterOpts) Counter { - desc := NewDesc( + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ) if opts.now == nil { opts.now = time.Now @@ -205,6 +206,7 @@ func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec { opts.Help, opts.VariableLabels, opts.ConstLabels, + WithUnit(opts.Unit), ) if opts.now == nil { opts.now = time.Now @@ -349,10 +351,11 @@ type CounterFunc interface { // // Check out the ExampleGaugeFunc examples for the similar GaugeFunc. func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc { - return newValueFunc(NewDesc( + return newValueFunc(V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ), CounterValue, function) } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/desc.go b/vendor/github.com/prometheus/client_golang/prometheus/desc.go index 2331b8b4f..a3c92e7a4 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/desc.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/desc.go @@ -47,6 +47,8 @@ type Desc struct { fqName string // help provides some helpful information about this metric. help string + // unit provides the unit of this metric. + unit string // constLabelPairs contains precalculated DTO label pairs based on // the constant labels. constLabelPairs []*dto.LabelPair @@ -66,6 +68,16 @@ type Desc struct { err error } +// DescOpt allows setting optional fields for NewDesc. +type DescOpt func(*Desc) + +// WithUnit sets the unit for a Desc. +func WithUnit(unit string) DescOpt { + return func(d *Desc) { + d.unit = unit + } +} + // NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc // and will be reported on registration time. variableLabels and constLabels can // be nil if no such labels should be set. fqName must not be empty. @@ -89,14 +101,17 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // // For constLabels, the label values are constant. Therefore, they are fully // specified in the Desc. See the Collector example for a usage pattern. -func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, constLabels Labels) *Desc { +func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, constLabels Labels, opts ...DescOpt) *Desc { d := &Desc{ fqName: fqName, help: help, variableLabels: variableLabels.compile(), } - //nolint:staticcheck // TODO: Don't use deprecated model.NameValidationScheme. - if !model.NameValidationScheme.IsValidMetricName(fqName) { + + for _, opt := range opts { + opt(d) + } + if !model.UTF8Validation.IsValidMetricName(fqName) { d.err = fmt.Errorf("%q is not a valid metric name", fqName) return d } @@ -150,11 +165,13 @@ func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, const d.id = xxh.Sum64() // Sort labelNames so that order doesn't matter for the hash. sort.Strings(labelNames) - // Now hash together (in this order) the help string and the sorted + // Now hash together (in this order) the help string, the unit string and the sorted // label names. xxh.Reset() xxh.WriteString(help) xxh.Write(separatorByteSlice) + xxh.WriteString(d.unit) + xxh.Write(separatorByteSlice) for _, labelName := range labelNames { xxh.WriteString(labelName) xxh.Write(separatorByteSlice) @@ -182,6 +199,15 @@ func NewInvalidDesc(err error) *Desc { } } +// Err returns an error that occurred during construction, if any. +// +// Calling this method is optional. It can be used to detect construction +// errors early, before invoking other methods on the Desc. If an error is +// present, later operations may not behave as expected. +func (d *Desc) Err() error { + return d.err +} + func (d *Desc) String() string { lpStrings := make([]string, 0, len(d.constLabelPairs)) for _, lp := range d.constLabelPairs { @@ -202,9 +228,10 @@ func (d *Desc) String() string { } } return fmt.Sprintf( - "Desc{fqName: %q, help: %q, constLabels: {%s}, variableLabels: {%s}}", + "Desc{fqName: %q, help: %q, unit: %q, constLabels: {%s}, variableLabels: {%s}}", d.fqName, d.help, + d.unit, strings.Join(lpStrings, ","), strings.Join(vlStrings, ","), ) diff --git a/vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go b/vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go index de5a85629..327746f43 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go @@ -47,14 +47,14 @@ func (e *expvarCollector) Collect(ch chan<- Metric) { if expVar == nil { continue } - var v interface{} + var v any labels := make([]string, len(desc.variableLabels.names)) if err := json.Unmarshal([]byte(expVar.String()), &v); err != nil { ch <- NewInvalidMetric(desc, err) continue } - var processValue func(v interface{}, i int) - processValue = func(v interface{}, i int) { + var processValue func(v any, i int) + processValue = func(v any, i int) { if i >= len(labels) { copiedLabels := append(make([]string, 0, len(labels)), labels...) switch v := v.(type) { @@ -72,7 +72,7 @@ func (e *expvarCollector) Collect(ch chan<- Metric) { ch <- m return } - vm, ok := v.(map[string]interface{}) + vm, ok := v.(map[string]any) if !ok { return } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/gauge.go b/vendor/github.com/prometheus/client_golang/prometheus/gauge.go index dd2eac940..41e54bf27 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/gauge.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/gauge.go @@ -76,11 +76,12 @@ type GaugeVecOpts struct { // scenarios for Gauges and Counters, where the former tends to be Set-heavy and // the latter Inc-heavy. func NewGauge(opts GaugeOpts) Gauge { - desc := NewDesc( + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ) result := &gauge{desc: desc, labelPairs: desc.constLabelPairs} result.init(result) // Init self-collection. @@ -163,6 +164,7 @@ func (v2) NewGaugeVec(opts GaugeVecOpts) *GaugeVec { opts.Help, opts.VariableLabels, opts.ConstLabels, + WithUnit(opts.Unit), ) return &GaugeVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { @@ -302,10 +304,11 @@ type GaugeFunc interface { // value of 1. Example: // https://github.com/prometheus/common/blob/8558a5b7db3c84fa38b4766966059a7bd5bfa2ee/version/info.go#L36-L56 func NewGaugeFunc(opts GaugeOpts, function func() float64) GaugeFunc { - return newValueFunc(NewDesc( + return newValueFunc(V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ), GaugeValue, function) } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_go116.go b/vendor/github.com/prometheus/client_golang/prometheus/go_collector_go116.go deleted file mode 100644 index 897a6e906..000000000 --- a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_go116.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !go1.17 -// +build !go1.17 - -package prometheus - -import ( - "runtime" - "sync" - "time" -) - -type goCollector struct { - base baseGoCollector - - // ms... are memstats related. - msLast *runtime.MemStats // Previously collected memstats. - msLastTimestamp time.Time - msMtx sync.Mutex // Protects msLast and msLastTimestamp. - msMetrics memStatsMetrics - msRead func(*runtime.MemStats) // For mocking in tests. - msMaxWait time.Duration // Wait time for fresh memstats. - msMaxAge time.Duration // Maximum allowed age of old memstats. -} - -// NewGoCollector is the obsolete version of collectors.NewGoCollector. -// See there for documentation. -// -// Deprecated: Use collectors.NewGoCollector instead. -func NewGoCollector() Collector { - msMetrics := goRuntimeMemStats() - msMetrics = append(msMetrics, struct { - desc *Desc - eval func(*runtime.MemStats) float64 - valType ValueType - }{ - // This metric is omitted in Go1.17+, see https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 - desc: NewDesc( - memstatNamespace("gc_cpu_fraction"), - "The fraction of this program's available CPU time used by the GC since the program started.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return ms.GCCPUFraction }, - valType: GaugeValue, - }) - return &goCollector{ - base: newBaseGoCollector(), - msLast: &runtime.MemStats{}, - msRead: runtime.ReadMemStats, - msMaxWait: time.Second, - msMaxAge: 5 * time.Minute, - msMetrics: msMetrics, - } -} - -// Describe returns all descriptions of the collector. -func (c *goCollector) Describe(ch chan<- *Desc) { - c.base.Describe(ch) - for _, i := range c.msMetrics { - ch <- i.desc - } -} - -// Collect returns the current state of all metrics of the collector. -func (c *goCollector) Collect(ch chan<- Metric) { - var ( - ms = &runtime.MemStats{} - done = make(chan struct{}) - ) - // Start reading memstats first as it might take a while. - go func() { - c.msRead(ms) - c.msMtx.Lock() - c.msLast = ms - c.msLastTimestamp = time.Now() - c.msMtx.Unlock() - close(done) - }() - - // Collect base non-memory metrics. - c.base.Collect(ch) - - timer := time.NewTimer(c.msMaxWait) - select { - case <-done: // Our own ReadMemStats succeeded in time. Use it. - timer.Stop() // Important for high collection frequencies to not pile up timers. - c.msCollect(ch, ms) - return - case <-timer.C: // Time out, use last memstats if possible. Continue below. - } - c.msMtx.Lock() - if time.Since(c.msLastTimestamp) < c.msMaxAge { - // Last memstats are recent enough. Collect from them under the lock. - c.msCollect(ch, c.msLast) - c.msMtx.Unlock() - return - } - // If we are here, the last memstats are too old or don't exist. We have - // to wait until our own ReadMemStats finally completes. For that to - // happen, we have to release the lock. - c.msMtx.Unlock() - <-done - c.msCollect(ch, ms) -} - -func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) { - for _, i := range c.msMetrics { - ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms)) - } -} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go b/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go index 6b8684731..1db1c4be0 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go @@ -98,7 +98,7 @@ type goCollector struct { // snapshot is always produced by Collect. mu sync.Mutex - // Contains all samples that has to retrieved from runtime/metrics (not all of them will be exposed). + // Contains all samples that have to be retrieved from runtime/metrics (not all of them will be exposed). sampleBuf []metrics.Sample // sampleMap allows lookup for MemStats metrics and runtime/metrics histograms for exact sums. sampleMap map[string]*metrics.Sample @@ -210,16 +210,26 @@ func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector { sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name}) sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1] + // Extract unit from the runtime/metrics name (e.g., "/gc/heap/allocs:bytes" -> "bytes") + // and sanitize to match Prometheus naming conventions (e.g., "cpu-seconds" -> "cpu_seconds") + var unit string + if idx := strings.IndexRune(d.Name, ':'); idx >= 0 { + unit = d.Name[idx+1:] + unit = strings.ReplaceAll(unit, "-", "_") + unit = strings.ReplaceAll(unit, "*", "_") + unit = strings.ReplaceAll(unit, "/", "_per_") + } + var m collectorMetric if d.Kind == metrics.KindFloat64Histogram { _, hasSum := opt.RuntimeMetricSumForHist[d.Name] - unit := d.Name[strings.IndexRune(d.Name, ':')+1:] m = newBatchHistogram( - NewDesc( + V2.NewDesc( BuildFQName(namespace, subsystem, name), help, + UnconstrainedLabels(nil), nil, - nil, + WithUnit(unit), ), internal.RuntimeMetricsBucketsForUnit(bucketsMap[d.Name], unit), hasSum, @@ -230,6 +240,7 @@ func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector { Subsystem: subsystem, Name: name, Help: help, + Unit: unit, }, ) } else { @@ -238,6 +249,7 @@ func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector { Subsystem: subsystem, Name: name, Help: help, + Unit: unit, }) } metricSet = append(metricSet, m) diff --git a/vendor/github.com/prometheus/client_golang/prometheus/histogram.go b/vendor/github.com/prometheus/client_golang/prometheus/histogram.go index c453b754a..88bae3b32 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/histogram.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/histogram.go @@ -378,6 +378,9 @@ type HistogramOpts struct { // string. Help string + // Unit provides the unit of this Histogram. + Unit string + // ConstLabels are used to attach fixed labels to this metric. Metrics // with the same fully-qualified name must have the same label names in // their ConstLabels. @@ -522,11 +525,12 @@ type HistogramVecOpts struct { // for each bucket. func NewHistogram(opts HistogramOpts) Histogram { return newHistogram( - NewDesc( + V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ), opts, ) @@ -966,7 +970,7 @@ func (h *histogram) maybeReset( // We are using the possibly mocked h.now() rather than // time.Since(h.lastResetTime) to enable testing. if h.nativeHistogramMinResetDuration == 0 || // No reset configured. - h.resetScheduled || // Do not interefere if a reset is already scheduled. + h.resetScheduled || // Do not interfere if a reset is already scheduled. h.now().Sub(h.lastResetTime) < h.nativeHistogramMinResetDuration { return false } @@ -1053,8 +1057,8 @@ func (h *histogram) maybeWidenZeroBucket(hot, cold *histogramCounts) bool { atomic.StoreUint64(&cold.nativeHistogramZeroThresholdBits, math.Float64bits(newZeroThreshold)) // ...and then merge the newly deleted buckets into the wider zero // bucket. - mergeAndDeleteOrAddAndReset := func(hotBuckets, coldBuckets *sync.Map) func(k, v interface{}) bool { - return func(k, v interface{}) bool { + mergeAndDeleteOrAddAndReset := func(hotBuckets, coldBuckets *sync.Map) func(k, v any) bool { + return func(k, v any) bool { key := k.(int) bucket := v.(*int64) if key == smallestKey { @@ -1107,8 +1111,8 @@ func (h *histogram) doubleBucketWidth(hot, cold *histogramCounts) { // ...adjust the schema in the cold counts, too... atomic.StoreInt32(&cold.nativeHistogramSchema, coldSchema) // ...and then merge the cold buckets into the wider hot buckets. - merge := func(hotBuckets *sync.Map) func(k, v interface{}) bool { - return func(k, v interface{}) bool { + merge := func(hotBuckets *sync.Map) func(k, v any) bool { + return func(k, v any) bool { key := k.(int) bucket := v.(*int64) // Adjust key to match the bucket to merge into. @@ -1190,6 +1194,7 @@ func (v2) NewHistogramVec(opts HistogramVecOpts) *HistogramVec { opts.Help, opts.VariableLabels, opts.ConstLabels, + WithUnit(opts.Unit), ) return &HistogramVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { @@ -1476,7 +1481,7 @@ func pickSchema(bucketFactor float64) int32 { func makeBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) { var ii []int - buckets.Range(func(k, v interface{}) bool { + buckets.Range(func(k, v any) bool { ii = append(ii, k.(int)) return true }) @@ -1553,8 +1558,8 @@ func addToBucket(buckets *sync.Map, key int, increment int64) bool { // according to the buckets ranged through. It then resets all buckets ranged // through to 0 (but leaves them in place so that they don't need to get // recreated on the next scrape). -func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v interface{}) bool { - return func(k, v interface{}) bool { +func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v any) bool { + return func(k, v any) bool { bucket := v.(*int64) if addToBucket(hotBuckets, k.(int), atomic.LoadInt64(bucket)) { atomic.AddUint32(bucketNumber, 1) @@ -1565,7 +1570,7 @@ func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v interface } func deleteSyncMap(m *sync.Map) { - m.Range(func(k, v interface{}) bool { + m.Range(func(k, v any) bool { m.Delete(k) return true }) @@ -1573,7 +1578,7 @@ func deleteSyncMap(m *sync.Map) { func findSmallestKey(m *sync.Map) int { result := math.MaxInt32 - m.Range(func(k, v interface{}) bool { + m.Range(func(k, v any) bool { key := k.(int) if key < result { result = key diff --git a/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go b/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go index 7bac0da33..2db270f21 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go @@ -78,7 +78,7 @@ type OpCode struct { // notion, pairing up elements that appear uniquely in each sequence. // That, and the method here, appear to yield more intuitive difference // reports than does diff. This method appears to be the least vulnerable -// to synching up on blocks of "junk lines", though (like blank lines in +// to syncing up on blocks of "junk lines", though (like blank lines in // ordinary text files, or maybe "

" lines in HTML files). That may be // because this is the only method of the 3 that has a *concept* of // "junk" . @@ -567,7 +567,7 @@ type UnifiedDiff struct { func WriteUnifiedDiff(writer io.Writer, diff UnifiedDiff) error { buf := bufio.NewWriter(writer) defer buf.Flush() - wf := func(format string, args ...interface{}) error { + wf := func(format string, args ...any) error { _, err := fmt.Fprintf(buf, format, args...) return err } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/labels.go b/vendor/github.com/prometheus/client_golang/prometheus/labels.go index 5fe8d3b4d..a0285489a 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/labels.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/labels.go @@ -184,6 +184,5 @@ func validateLabelValues(vals []string, expectedNumberOfValues int) error { } func checkLabelName(l string) bool { - //nolint:staticcheck // TODO: Don't use deprecated model.NameValidationScheme. - return model.NameValidationScheme.IsValidLabelName(l) && !strings.HasPrefix(l, reservedLabelPrefix) + return model.UTF8Validation.IsValidLabelName(l) && !strings.HasPrefix(l, reservedLabelPrefix) } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/metric.go b/vendor/github.com/prometheus/client_golang/prometheus/metric.go index 76e59f128..c5cb90adf 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/metric.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/metric.go @@ -81,6 +81,9 @@ type Opts struct { // string. Help string + // Unit provides the unit of this metric as per https://prometheus.io/docs/specs/om + Unit string + // ConstLabels are used to attach fixed labels to this metric. Metrics // with the same fully-qualified name must have the same label names in // their ConstLabels. diff --git a/vendor/github.com/prometheus/client_golang/prometheus/process_collector_darwin.go b/vendor/github.com/prometheus/client_golang/prometheus/process_collector_darwin.go index b32c95fa3..2b16298f4 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/process_collector_darwin.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/process_collector_darwin.go @@ -72,7 +72,13 @@ func getOpenFileCount() (float64, error) { } func (c *processCollector) processCollect(ch chan<- Metric) { - if procs, err := unix.SysctlKinfoProcSlice("kern.proc.pid", os.Getpid()); err == nil { + pid, err := c.pidFn() + if err != nil { + c.reportError(ch, nil, err) + return + } + + if procs, err := unix.SysctlKinfoProcSlice("kern.proc.pid", pid); err == nil { if len(procs) == 1 { startTime := float64(procs[0].Proc.P_starttime.Nano() / 1e9) ch <- MustNewConstMetric(c.startTime, GaugeValue, startTime) @@ -84,6 +90,11 @@ func (c *processCollector) processCollect(ch chan<- Metric) { c.reportError(ch, c.startTime, err) } + if pid != os.Getpid() { + c.reportError(ch, nil, fmt.Errorf("collecting metrics for pid %d is not supported on darwin: process metrics collection is limited to the current process (pid %d)", pid, os.Getpid())) + return + } + // The proc structure returned by kern.proc.pid above has an Rusage member, // but it is not filled in, so it needs to be fetched by getrusage(2). For // that call, the UTime, STime, and Maxrss members are filled out, but not diff --git a/vendor/github.com/prometheus/client_golang/prometheus/process_collector_windows.go b/vendor/github.com/prometheus/client_golang/prometheus/process_collector_windows.go index fa474289e..c08dd05f0 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/process_collector_windows.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/process_collector_windows.go @@ -30,6 +30,10 @@ var ( procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo") procGetProcessHandleCount = modkernel32.NewProc("GetProcessHandleCount") + + openProcess = windows.OpenProcess + closeHandle = windows.CloseHandle + getProcessTimes = windows.GetProcessTimes ) type processMemoryCounters struct { @@ -79,10 +83,21 @@ func getProcessHandleCount(handle windows.Handle) (uint32, error) { } func (c *processCollector) processCollect(ch chan<- Metric) { - h := windows.CurrentProcess() + pid, err := c.pidFn() + if err != nil { + c.reportError(ch, nil, err) + return + } + + h, err := openProcess(windows.PROCESS_QUERY_INFORMATION|windows.PROCESS_VM_READ, false, uint32(pid)) + if err != nil { + c.reportError(ch, nil, err) + return + } + defer closeHandle(h) var startTime, exitTime, kernelTime, userTime windows.Filetime - err := windows.GetProcessTimes(h, &startTime, &exitTime, &kernelTime, &userTime) + err = getProcessTimes(h, &startTime, &exitTime, &kernelTime, &userTime) if err != nil { c.reportError(ch, nil, err) return diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go index 763d99e36..c28af5ce2 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go @@ -37,10 +37,12 @@ import ( "fmt" "io" "net/http" + "slices" "strconv" "sync" "time" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil" @@ -74,11 +76,118 @@ func defaultCompressionFormats() []Compression { } var gzipPool = sync.Pool{ - New: func() interface{} { + New: func() any { return gzip.NewWriter(nil) }, } +// coalescingGatherer wraps a TransactionalGatherer to deduplicate concurrent +// Gather calls. When a Gather is already in flight, new callers join the +// existing cycle and receive the same result once it completes. The underlying +// done function is called exactly once, when the last joined caller releases. +// +// This prevents goroutine pile-up when the scrape rate is faster than the +// time collectors need to produce metrics. +type coalescingGatherer struct { + g prometheus.TransactionalGatherer + mu sync.Mutex + cycle *gatherCycle +} + +// gatherCycle tracks a single in-flight Gather and all HTTP handlers sharing it. +type gatherCycle struct { + ready chan struct{} // closed when Gather completes; happens-before reads of mfs/err/done + mfs []*dto.MetricFamily // canonical result, set before ready is closed; callers get a slices.Clone, the element values stay shared and must not be mutated + err error // set before ready is closed + done func() // underlying done callback; set before ready is closed + refs int // number of handlers using this cycle; protected by coalescingGatherer.mu +} + +var _ prometheus.TransactionalGatherer = (*coalescingGatherer)(nil) // compile-time interface check + +// errGatherPanicked is returned to callers that joined an in-flight coalesced +// Gather whose underlying gatherer panicked. See the panic guard in Gather for +// why joiners receive this error instead of the panic itself. +var errGatherPanicked = errors.New("coalesced gather panicked") + +func (c *coalescingGatherer) Gather() ([]*dto.MetricFamily, func(), error) { + c.mu.Lock() + if cy := c.cycle; cy != nil { + // c.cycle is non-nil while Gather runs or handlers are still consuming its results. + cy.refs++ + c.mu.Unlock() + <-cy.ready + // Each caller gets its own slice header so it can filter or reorder + // without racing other callers sharing this cycle. The *dto.MetricFamily + // values remain shared and must not be mutated in place. + return slices.Clone(cy.mfs), c.releaseFunc(cy), cy.err + } + cy := &gatherCycle{ + ready: make(chan struct{}), + done: func() {}, + refs: 1, + } + c.cycle = cy + c.mu.Unlock() + + // Guard against a panic in c.g.Gather. The common case, a panicking + // Collector, never reaches here: Registry.Gather recovers Collector panics + // and returns them as an error. This guard only covers the rare case where + // the wrapped gatherer itself panics. + // + // We deliberately do not recover: the leader's panic propagates and is + // handled by net/http exactly as it would be without coalescing. We only + // set cy.err before closing cy.ready so joiners waiting on <-cy.ready fail + // with that error instead of silently returning an empty, successful + // response, and we clear c.cycle so the next Gather starts a fresh cycle. + // + // The leader never runs its own releaseFunc on this path, so its ref is + // not decremented; that is harmless because the cycle is detached (c.cycle + // = nil) and cy.done is still the no-op set at construction (c.g.Gather + // panicked before assigning a real done). If cy.done is ever made non-nil + // before c.g.Gather runs, this path would need to release it. + panicked := true + defer func() { + if panicked { + c.mu.Lock() + if c.cycle == cy { + c.cycle = nil + } + c.mu.Unlock() + cy.err = errGatherPanicked // set before close: happens-before joiners' reads + close(cy.ready) + } + }() + cy.mfs, cy.done, cy.err = c.g.Gather() + panicked = false + close(cy.ready) // happens-before joiners' reads of cy.mfs/err/done + + // Clone here too so cy.mfs stays the write-once canonical slice: joiners + // read it concurrently via slices.Clone, so the leader must not hand out + // (and potentially reorder) the same backing array. + return slices.Clone(cy.mfs), c.releaseFunc(cy), cy.err +} + +// releaseFunc returns the done callback for one caller sharing cy. +// When the last caller releases, the underlying done is invoked and the +// cycle is cleared so the next Gather starts fresh. +func (c *coalescingGatherer) releaseFunc(cy *gatherCycle) func() { + return func() { + c.mu.Lock() + cy.refs-- + if cy.refs > 0 { + c.mu.Unlock() + return + } + // Last caller. + if c.cycle == cy { + c.cycle = nil + } + c.mu.Unlock() + cy.done() // called outside the lock to avoid holding it during done + } +} + // Handler returns an http.Handler for the prometheus.DefaultGatherer, using // default HandlerOpts, i.e. it reports the first error as an HTTP error, it has // no error logging, and it applies compression if requested by the client. @@ -89,6 +198,10 @@ var gzipPool = sync.Pool{ // metrics used for instrumentation will be shared between them, providing // global scrape counts. // +// The handler supports filtering metrics by name using the `name[]` query parameter. +// Multiple metric names can be specified by providing the parameter multiple times. +// When no name[] parameters are provided, all metrics are returned. +// // This function is meant to cover the bulk of basic use cases. If you are doing // anything that requires more customization (including using a non-default // Gatherer, different instrumentation, and non-default HandlerOpts), use the @@ -105,6 +218,10 @@ func Handler() http.Handler { // Gatherers, with non-default HandlerOpts, and/or with custom (or no) // instrumentation. Use the InstrumentMetricHandler function to apply the same // kind of instrumentation as it is used by the Handler function. +// +// The handler supports filtering metrics by name using the `name[]` query parameter. +// Multiple metric names can be specified by providing the parameter multiple times. +// When no name[] parameters are provided, all metrics are returned. func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { return HandlerForTransactional(prometheus.ToTransactionalGatherer(reg), opts) } @@ -112,7 +229,15 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { // HandlerForTransactional is like HandlerFor, but it uses transactional gather, which // can safely change in-place returned *dto.MetricFamily before call to `Gather` and after // call to `done` of that `Gather`. +// +// The handler supports filtering metrics by name using the `name[]` query parameter. +// Multiple metric names can be specified by providing the parameter multiple times. +// When no name[] parameters are provided, all metrics are returned. func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerOpts) http.Handler { + if opts.CoalesceGather { + reg = &coalescingGatherer{g: reg} + } + var ( inFlightSem chan struct{} errCnt = prometheus.NewCounterVec( @@ -214,12 +339,14 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO rsp.Header().Set(contentEncodingHeader, encodingHeader) } - var enc expfmt.Encoder + var ( + enc expfmt.Encoder + encOpts []expfmt.EncoderOption + ) if opts.EnableOpenMetricsTextCreatedSamples { - enc = expfmt.NewEncoder(w, contentType, expfmt.WithCreatedLines()) - } else { - enc = expfmt.NewEncoder(w, contentType) + encOpts = append(encOpts, expfmt.WithCreatedLines()) } + enc = expfmt.NewEncoder(w, contentType, encOpts...) // handleError handles the error according to opts.ErrorHandling // and returns true if we have to abort after the handling. @@ -245,7 +372,24 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO return false } + // Build metric name filter set from query params (if any). The URL + // can be nil on hand-constructed requests. + var metricFilter map[string]struct{} + if req.URL != nil { + if metricNames := req.URL.Query()["name[]"]; len(metricNames) > 0 { + metricFilter = make(map[string]struct{}, len(metricNames)) + for _, name := range metricNames { + metricFilter[name] = struct{}{} + } + } + } + for _, mf := range mfs { + if metricFilter != nil { + if _, ok := metricFilter[mf.GetName()]; !ok { + continue + } + } if handleError(enc.Encode(mf)) { return } @@ -353,7 +497,7 @@ const ( // log.Logger from the standard library implements this interface, and it is // easy to implement by custom loggers, if they don't do so already anyway. type Logger interface { - Println(v ...interface{}) + Println(v ...any) } // HandlerOpts specifies options how to serve metrics via an http.Handler. The @@ -400,6 +544,40 @@ type HandlerOpts struct { // Service Unavailable and a suitable message in the body. If // MaxRequestsInFlight is 0 or negative, no limit is applied. MaxRequestsInFlight int + // CoalesceGather, if true, deduplicates concurrent Gather calls so that + // only one collection runs at a time. Additional requests that arrive + // while a Gather is in flight will receive the same result once it + // completes. This prevents goroutine pile-up when the scrape rate is + // faster than the time collectors need to produce metrics. + // + // When enabled, concurrent scrapers share a single metric snapshot per + // collection cycle. Each request receives its own copy of the returned + // slice, so filtering or reordering it (for example via name[] query + // parameters) is safe. The pointed-to MetricFamily values are still + // shared: the built-in handler only reads them, so this is safe in + // practice, but a custom TransactionalGatherer that mutates the returned + // families in place after Gather returns must not use this option. + // + // Because the snapshot is shared, a request that arrives while a cycle is + // in flight receives that cycle's result even though collection began + // before the request; two scrapers joined to one cycle observe the same + // timestamps rather than independently gathered data. + // + // Consider using CoalesceGather together with Timeout. Timeout bounds the + // client-facing response time and keeps at most one collection running at + // a time, but it does not cancel the underlying Gather: a joined request + // that times out still holds a MaxRequestsInFlight slot until the shared + // collection completes. + // + // Panic handling: a panicking Collector is already turned into an error by + // the registry, so joiners receive that error like any other. In the rare + // case where the wrapped gatherer itself panics, the panicking request's + // panic propagates as usual (handled by net/http), while requests that + // joined the same cycle receive an error rather than an empty response. + // + // NOTE: This option is experimental and may change or be removed in a + // future release. + CoalesceGather bool // If handling a request takes longer than Timeout, it is responded to // with 503 ServiceUnavailable and a suitable Message. No timeout is // applied if Timeout is 0 or negative. Note that with the current @@ -407,8 +585,9 @@ type HandlerOpts struct { // described above (and even that only if sending of the body hasn't // started yet), while the bulk work of gathering all the metrics keeps // running in the background (with the eventual result to be thrown - // away). Until the implementation is improved, it is recommended to - // implement a separate timeout in potentially slow Collectors. + // away). When CoalesceGather is enabled, only one such background Gather + // can be in flight at a time. It is also recommended to implement a + // separate timeout in potentially slow Collectors. Timeout time.Duration // If true, the experimental OpenMetrics encoding is added to the // possible options during content negotiation. Note that Prometheus @@ -460,7 +639,7 @@ func httpError(rsp http.ResponseWriter, err error) { // negotiateEncodingWriter reads the Accept-Encoding header from a request and // selects the right compression based on an allow-list of supported -// compressions. It returns a writer implementing the compression and an the +// compressions. It returns a writer implementing the compression and the // correct value that the caller can set in the response header. func negotiateEncodingWriter(r *http.Request, rw io.Writer, compressions []string) (_ io.Writer, encodingHeaderValue string, closeWriter func(), _ error) { if len(compressions) == 0 { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go index d3482c40c..024857974 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go @@ -75,10 +75,10 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou resp, err := next.RoundTrip(r) if err == nil { l := labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...) - for label, resolve := range rtOpts.extraLabelsFromCtx { - l[label] = resolve(resp.Request.Context()) + for label, resolve := range rtOpts.extraLabelsFromRequest { + l[label] = resolve(resp.Request) } - addWithExemplar(counter.With(l), 1, rtOpts.getExemplarFn(r.Context())) + addWithExemplar(counter.With(l), 1, rtOpts.getExemplarFn(r)) } return resp, err } @@ -119,10 +119,10 @@ func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundT resp, err := next.RoundTrip(r) if err == nil { l := labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...) - for label, resolve := range rtOpts.extraLabelsFromCtx { - l[label] = resolve(resp.Request.Context()) + for label, resolve := range rtOpts.extraLabelsFromRequest { + l[label] = resolve(resp.Request) } - observeWithExemplar(obs.With(l), time.Since(start).Seconds(), rtOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), time.Since(start).Seconds(), rtOpts.getExemplarFn(r)) } return resp, err } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go index 9332b0249..9dec091ac 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go @@ -28,24 +28,36 @@ import ( // magicString is used for the hacky label test in checkLabels. Remove once fixed. const magicString = "zZgWfBxLqvG8kc8IMv3POi2Bb0tZI3vAnBx+gBaFi9FyPzB/CzKUer1yufDa" -// observeWithExemplar is a wrapper for [prometheus.ExemplarAdder.ExemplarObserver], -// which falls back to [prometheus.Observer.Observe] if no labels are provided. +// observeWithExemplar records val on obs. If labels is non-nil and obs +// implements [prometheus.ExemplarObserver], the exemplar is attached via +// ObserveWithExemplar; otherwise the exemplar is dropped and the value is +// recorded with a plain [prometheus.Observer.Observe]. This mirrors the +// safe-cast pattern in [prometheus.Timer.ObserveDurationWithExemplar] and +// ensures we never panic when callers pass an ObserverVec backed by a +// summary, which cannot carry exemplars in the Prometheus exposition format. func observeWithExemplar(obs prometheus.Observer, val float64, labels map[string]string) { - if labels == nil { - obs.Observe(val) - return + if labels != nil { + if eo, ok := obs.(prometheus.ExemplarObserver); ok { + eo.ObserveWithExemplar(val, labels) + return + } } - obs.(prometheus.ExemplarObserver).ObserveWithExemplar(val, labels) + obs.Observe(val) } -// addWithExemplar is a wrapper for [prometheus.ExemplarAdder.AddWithExemplar], -// which falls back to [prometheus.Counter.Add] if no labels are provided. -func addWithExemplar(obs prometheus.Counter, val float64, labels map[string]string) { - if labels == nil { - obs.Add(val) - return +// addWithExemplar records val on c. If labels is non-nil and c implements +// [prometheus.ExemplarAdder], the exemplar is attached via AddWithExemplar; +// otherwise the exemplar is dropped and the value is recorded with a plain +// [prometheus.Counter.Add]. The safe-cast keeps the helper robust against +// custom Counter implementations that do not advertise exemplar support. +func addWithExemplar(c prometheus.Counter, val float64, labels map[string]string) { + if labels != nil { + if ea, ok := c.(prometheus.ExemplarAdder); ok { + ea.AddWithExemplar(val, labels) + return + } } - obs.(prometheus.ExemplarAdder).AddWithExemplar(val, labels) + c.Add(val) } // InstrumentHandlerInFlight is a middleware that wraps the provided @@ -97,10 +109,10 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op next.ServeHTTP(d, r) l := labels(code, method, r.Method, d.Status(), hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r)) } } @@ -108,10 +120,10 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op now := time.Now() next.ServeHTTP(w, r) l := labels(code, method, r.Method, 0, hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r)) } } @@ -147,10 +159,10 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, next.ServeHTTP(d, r) l := labels(code, method, r.Method, d.Status(), hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - addWithExemplar(counter.With(l), 1, hOpts.getExemplarFn(r.Context())) + addWithExemplar(counter.With(l), 1, hOpts.getExemplarFn(r)) } } @@ -158,10 +170,10 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, next.ServeHTTP(w, r) l := labels(code, method, r.Method, 0, hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - addWithExemplar(counter.With(l), 1, hOpts.getExemplarFn(r.Context())) + addWithExemplar(counter.With(l), 1, hOpts.getExemplarFn(r)) } } @@ -200,10 +212,10 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha now := time.Now() d := newDelegator(w, func(status int) { l := labels(code, method, r.Method, status, hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), time.Since(now).Seconds(), hOpts.getExemplarFn(r)) }) next.ServeHTTP(d, r) } @@ -244,10 +256,10 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, size := computeApproximateRequestSize(r) l := labels(code, method, r.Method, d.Status(), hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), float64(size), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), float64(size), hOpts.getExemplarFn(r)) } } @@ -256,10 +268,10 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, size := computeApproximateRequestSize(r) l := labels(code, method, r.Method, 0, hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), float64(size), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), float64(size), hOpts.getExemplarFn(r)) } } @@ -296,10 +308,10 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler next.ServeHTTP(d, r) l := labels(code, method, r.Method, d.Status(), hOpts.extraMethods...) - for label, resolve := range hOpts.extraLabelsFromCtx { - l[label] = resolve(r.Context()) + for label, resolve := range hOpts.extraLabelsFromRequest { + l[label] = resolve(r) } - observeWithExemplar(obs.With(l), float64(d.Written()), hOpts.getExemplarFn(r.Context())) + observeWithExemplar(obs.With(l), float64(d.Written()), hOpts.getExemplarFn(r)) }) } @@ -366,7 +378,7 @@ func checkLabels(c prometheus.Collector) (code, method bool) { panic("metric partitioned with non-supported labels") } } - return + return code, method } func isLabelCurried(c prometheus.Collector, label string) bool { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/option.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/option.go index 5d4383aa1..d4c0954f3 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/option.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/option.go @@ -15,6 +15,7 @@ package promhttp import ( "context" + "net/http" "github.com/prometheus/client_golang/prometheus" ) @@ -24,28 +25,31 @@ type Option interface { apply(*options) } +// LabelValueFromRequest is used to compute the label value from request. +type LabelValueFromRequest func(request *http.Request) string + // LabelValueFromCtx are used to compute the label value from request context. // Context can be filled with values from request through middleware. type LabelValueFromCtx func(ctx context.Context) string // options store options for both a handler or round tripper. type options struct { - extraMethods []string - getExemplarFn func(requestCtx context.Context) prometheus.Labels - extraLabelsFromCtx map[string]LabelValueFromCtx + extraMethods []string + getExemplarFn func(req *http.Request) prometheus.Labels + extraLabelsFromRequest map[string]LabelValueFromRequest } func defaultOptions() *options { return &options{ - getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }, - extraLabelsFromCtx: map[string]LabelValueFromCtx{}, + getExemplarFn: func(req *http.Request) prometheus.Labels { return nil }, + extraLabelsFromRequest: map[string]LabelValueFromRequest{}, } } func (o *options) emptyDynamicLabels() prometheus.Labels { labels := prometheus.Labels{} - for label := range o.extraLabelsFromCtx { + for label := range o.extraLabelsFromRequest { labels[label] = "" } @@ -66,19 +70,39 @@ func WithExtraMethods(methods ...string) Option { }) } -// WithExemplarFromContext allows to inject function that will get exemplar from context that will be put to counter and histogram metrics. +// WithExemplarFromRequest allows you to inject a function that will get exemplar from request that will be put to counter and histogram metrics. // If the function returns nil labels or the metric does not support exemplars, no exemplar will be added (noop), but // metric will continue to observe/increment. -func WithExemplarFromContext(getExemplarFn func(requestCtx context.Context) prometheus.Labels) Option { +func WithExemplarFromRequest(getExemplarFn func(req *http.Request) prometheus.Labels) Option { return optionApplyFunc(func(o *options) { o.getExemplarFn = getExemplarFn }) } +// WithExemplarFromContext allows you to inject a function that will get exemplar from context that will be put to counter and histogram metrics. +// If the function returns nil labels or the metric does not support exemplars, no exemplar will be added (noop), but +// metric will continue to observe/increment. +func WithExemplarFromContext(getExemplarFn func(requestCtx context.Context) prometheus.Labels) Option { + return optionApplyFunc(func(o *options) { + o.getExemplarFn = func(req *http.Request) prometheus.Labels { + return getExemplarFn(req.Context()) + } + }) +} + +// WithLabelFromRequest registers a label for dynamic resolution with access to the request. +func WithLabelFromRequest(name string, valueFn LabelValueFromRequest) Option { + return optionApplyFunc(func(o *options) { + o.extraLabelsFromRequest[name] = valueFn + }) +} + // WithLabelFromCtx registers a label for dynamic resolution with access to context. // See the example for ExampleInstrumentHandlerWithLabelResolver for example usage func WithLabelFromCtx(name string, valueFn LabelValueFromCtx) Option { return optionApplyFunc(func(o *options) { - o.extraLabelsFromCtx[name] = valueFn + o.extraLabelsFromRequest[name] = func(req *http.Request) string { + return valueFn(req.Context()) + } }) } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/registry.go b/vendor/github.com/prometheus/client_golang/prometheus/registry.go index c6fd2f58b..ed0681c8b 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/registry.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/registry.go @@ -214,6 +214,19 @@ func (err AlreadyRegisteredError) Error() string { // by a Gatherer to report multiple errors during MetricFamily gathering. type MultiError []error +// SafeMultiError is a thread-safe wrapper around MultiError using a mutex. +type SafeMultiError struct { + mu sync.Mutex + errs MultiError +} + +// Appends the provided error to the contained MultiError in a thread-safe way. +func (s *SafeMultiError) Append(err error) { + s.mu.Lock() + s.errs.Append(err) + s.mu.Unlock() +} + // Error formats the contained errors as a bullet point list, preceded by the // total number of errors. Note that this results in a multi-line string. func (errs MultiError) Error() string { @@ -408,6 +421,16 @@ func (r *Registry) MustRegister(cs ...Collector) { } } +// MustGather implements Gatherer. +// Wraps around Gather and panics if Gather fails for any reason. +func (r *Registry) MustGather() []*dto.MetricFamily { + mfs, err := r.Gather() + if err != nil { + panic(err) + } + return mfs +} + // Gather implements Gatherer. func (r *Registry) Gather() ([]*dto.MetricFamily, error) { r.mtx.RLock() @@ -423,7 +446,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { uncheckedMetricChan = make(chan Metric, capMetricChan) metricHashes = map[uint64]struct{}{} wg sync.WaitGroup - errs MultiError // The collected errors to return in the end. + safeErrs = &SafeMultiError{} // To collect errors in a threadsafe way registeredDescIDs map[uint64]struct{} // Only used for pedantic checks ) @@ -453,9 +476,9 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { for { select { case collector := <-checkedCollectors: - collector.Collect(checkedMetricChan) + safeErrs.Append((safeCollect(collector, checkedMetricChan))) case collector := <-uncheckedCollectors: - collector.Collect(uncheckedMetricChan) + safeErrs.Append(safeCollect(collector, uncheckedMetricChan)) default: return } @@ -499,7 +522,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { cmc = nil break } - errs.Append(processMetric( + safeErrs.Append(processMetric( metric, metricFamiliesByName, metricHashes, registeredDescIDs, @@ -509,7 +532,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { umc = nil break } - errs.Append(processMetric( + safeErrs.Append(processMetric( metric, metricFamiliesByName, metricHashes, nil, @@ -526,7 +549,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { cmc = nil break } - errs.Append(processMetric( + safeErrs.Append(processMetric( metric, metricFamiliesByName, metricHashes, registeredDescIDs, @@ -536,7 +559,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { umc = nil break } - errs.Append(processMetric( + safeErrs.Append(processMetric( metric, metricFamiliesByName, metricHashes, nil, @@ -556,7 +579,8 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { break } } - return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() + + return internal.NormalizeMetricFamilies(metricFamiliesByName), safeErrs.errs.MaybeUnwrap() } // Describe implements Collector. @@ -571,6 +595,24 @@ func (r *Registry) Describe(ch chan<- *Desc) { } } +// Helper wrapper around Collector.Collect. +// It tries to collect from the channel, recovers on panic and +// if it has recovered from a panic, then it sends an InvalidMetric into +// the channel with an InvalidDesc, and an error that includes a stack trace. +func safeCollect(c Collector, ch chan<- Metric) (err error) { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 64<<10) // 64 KB + n := runtime.Stack(buf, false) + err = fmt.Errorf("prometheus collector panic recovered: type=%T: error=%v\nstack trace=%s", c, r, buf[:n]) + ch <- NewInvalidMetric(NewInvalidDesc(err), err) + } + }() + c.Collect(ch) + + return err +} + // Collect implements Collector. func (r *Registry) Collect(ch chan<- Metric) { r.mtx.RLock() @@ -599,10 +641,12 @@ func WriteToTextfile(filename string, g Gatherer) error { mfs, err := g.Gather() if err != nil { + tmp.Close() return err } for _, mf := range mfs { if _, err := expfmt.MetricFamilyToText(tmp, mf); err != nil { + tmp.Close() return err } } @@ -685,6 +729,9 @@ func processMetric( metricFamily = &dto.MetricFamily{} metricFamily.Name = proto.String(desc.fqName) metricFamily.Help = proto.String(desc.help) + if desc.unit != "" { + metricFamily.Unit = proto.String(desc.unit) + } // TODO(beorn7): Simplify switch once Desc has type. switch { case dtoMetric.Gauge != nil: diff --git a/vendor/github.com/prometheus/client_golang/prometheus/summary.go b/vendor/github.com/prometheus/client_golang/prometheus/summary.go index ac5203c6f..c12b8d13d 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/summary.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/summary.go @@ -101,6 +101,9 @@ type SummaryOpts struct { // string. Help string + // Unit provides the unit of this Summary. + Unit string + // ConstLabels are used to attach fixed labels to this metric. Metrics // with the same fully-qualified name must have the same label names in // their ConstLabels. @@ -181,11 +184,12 @@ type SummaryVecOpts struct { // NewSummary creates a new Summary based on the provided SummaryOpts. func NewSummary(opts SummaryOpts) Summary { return newSummary( - NewDesc( + V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - nil, + UnconstrainedLabels(nil), opts.ConstLabels, + WithUnit(opts.Unit), ), opts, ) @@ -578,6 +582,7 @@ func (v2) NewSummaryVec(opts SummaryVecOpts) *SummaryVec { opts.Help, opts.VariableLabels, opts.ConstLabels, + WithUnit(opts.Unit), ) return &SummaryVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/timer.go b/vendor/github.com/prometheus/client_golang/prometheus/timer.go index 52344fef5..c1318ffb5 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/timer.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/timer.go @@ -37,10 +37,10 @@ type Timer struct { // or // // func TimeMeWithExemplar() { -// timer := NewTimer(myHistogram) -// defer timer.ObserveDurationWithExemplar(exemplar) -// // Do actual work. -// } +// timer := NewTimer(myHistogram) +// defer timer.ObserveDurationWithExemplar(exemplar) +// // Do actual work. +// } func NewTimer(o Observer) *Timer { return &Timer{ begin: time.Now(), @@ -66,7 +66,7 @@ func (t *Timer) ObserveDuration() time.Duration { // ObserveDurationWithExemplar is like ObserveDuration, but it will also // observe exemplar with the duration unless exemplar is nil or provided Observer can't -// be casted to ExemplarObserver. +// be cast to ExemplarObserver. func (t *Timer) ObserveDurationWithExemplar(exemplar Labels) time.Duration { d := time.Since(t.begin) eo, ok := t.observer.(ExemplarObserver) diff --git a/vendor/github.com/prometheus/client_golang/prometheus/vec.go b/vendor/github.com/prometheus/client_golang/prometheus/vec.go index 487b46656..121d2a963 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/vec.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/vec.go @@ -193,9 +193,11 @@ func (m *MetricVec) CurryWith(labels Labels) (*MetricVec, error) { // // Keeping the Metric for later use is possible (and should be considered if // performance is critical), but keep in mind that Reset, DeleteLabelValues and -// Delete can be used to delete the Metric from the MetricVec. In that case, the -// Metric will still exist, but it will not be exported anymore, even if a -// Metric with the same label values is created later. +// Delete can be used to delete the Metric from the MetricVec. In that case, if +// you have previously kept a reference to that Metric, the Metric object still +// exists and can be used, but it will not be exported anymore. If a Metric with +// the same label values is created later, updates to the old Metric reference +// will not be exported. // // An error is returned if the number of label values is not the same as the // number of variable labels in Desc (minus any curried labels). @@ -657,7 +659,7 @@ func inlineLabelValues(lvs []string, curry []curriedLabelValue) []string { } var labelsPool = &sync.Pool{ - New: func() interface{} { + New: func() any { return make(Labels) }, } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/wrap.go b/vendor/github.com/prometheus/client_golang/prometheus/wrap.go index 2ed128506..697f55558 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/wrap.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/wrap.go @@ -230,6 +230,7 @@ func wrapDesc(desc *Desc, prefix string, labels Labels) *Desc { return &Desc{ fqName: desc.fqName, help: desc.help, + unit: desc.unit, variableLabels: desc.variableLabels, constLabelPairs: desc.constLabelPairs, err: fmt.Errorf("attempted wrapping with already existing label name %q", ln), @@ -238,8 +239,8 @@ func wrapDesc(desc *Desc, prefix string, labels Labels) *Desc { constLabels[ln] = lv } // NewDesc will do remaining validations. - newDesc := V2.NewDesc(prefix+desc.fqName, desc.help, desc.variableLabels, constLabels) - // Propagate errors if there was any. This will override any errer + newDesc := V2.NewDesc(prefix+desc.fqName, desc.help, desc.variableLabels, constLabels, WithUnit(desc.unit)) + // Propagate errors if there was any. This will override any error // created by NewDesc above, i.e. earlier errors get precedence. if desc.err != nil { newDesc.err = desc.err diff --git a/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/cgo@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count b/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/cgo@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count deleted file mode 100644 index 09ce56ae8..000000000 Binary files a/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/cgo@go1.26.3-go1.26.3-darwin-arm64-2026-06-30.v1.count and /dev/null differ diff --git a/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/weekends b/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/weekends deleted file mode 100644 index 7ed6ff82d..000000000 --- a/vendor/github.com/prometheus/client_golang/prometheus/~/Library/Application Support/go/telemetry/local/weekends +++ /dev/null @@ -1 +0,0 @@ -5 diff --git a/vendor/github.com/prometheus/common/expfmt/expfmt.go b/vendor/github.com/prometheus/common/expfmt/expfmt.go index 4e4c13e72..10bf35708 100644 --- a/vendor/github.com/prometheus/common/expfmt/expfmt.go +++ b/vendor/github.com/prometheus/common/expfmt/expfmt.go @@ -122,7 +122,7 @@ func NewOpenMetricsFormat(version string) (Format, error) { // removed. func (f Format) WithEscapingScheme(s model.EscapingScheme) Format { var terms []string - for _, p := range strings.Split(string(f), ";") { + for p := range strings.SplitSeq(string(f), ";") { toks := strings.Split(p, "=") if len(toks) != 2 { trimmed := strings.TrimSpace(p) @@ -194,7 +194,7 @@ func (f Format) FormatType() FormatType { // "escaping" term exists, that will be used. Otherwise, the global default will // be returned. func (f Format) ToEscapingScheme() model.EscapingScheme { - for _, p := range strings.Split(string(f), ";") { + for p := range strings.SplitSeq(string(f), ";") { toks := strings.Split(p, "=") if len(toks) != 2 { continue diff --git a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go index 21b93bca3..0480e7af5 100644 --- a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go +++ b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go @@ -30,7 +30,6 @@ import ( type encoderOption struct { withCreatedLines bool - withUnit bool } type EncoderOption func(*encoderOption) @@ -51,17 +50,6 @@ func WithCreatedLines() EncoderOption { } } -// WithUnit is an EncoderOption enabling a set unit to be written to the output -// and to be added to the metric name, if it's not there already, as a suffix. -// Without opting in this way, the unit will not be added to the metric name and, -// on top of that, the unit will not be passed onto the output, even if it -// were declared in the *dto.MetricFamily struct, i.e. even if in.Unit !=nil. -func WithUnit() EncoderOption { - return func(t *encoderOption) { - t.withUnit = true - } -} - // MetricFamilyToOpenMetrics converts a MetricFamily proto message into the // OpenMetrics text format and writes the resulting lines to 'out'. It returns // the number of bytes written and any error encountered. The output will have @@ -99,15 +87,6 @@ func WithUnit() EncoderOption { // its type will be set to `unknown` in that case to avoid invalid OpenMetrics // output. // -// - According to the OM specs, the `# UNIT` line is optional, but if populated, -// the unit has to be present in the metric name as its suffix: -// (see https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#unit). -// However, in order to accommodate any potential scenario where such a change in the -// metric name is not desirable, the users are here given the choice of either explicitly -// opt in, in case they wish for the unit to be included in the output AND in the metric name -// as a suffix (see the description of the WithUnit function above), -// or not to opt in, in case they don't want for any of that to happen. -// // - No support for the following (optional) features: info type, // stateset type, gaugehistogram type. // @@ -151,9 +130,6 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...E if metricType == dto.MetricType_COUNTER && strings.HasSuffix(compliantName, "_total") { compliantName = name[:len(name)-6] } - if toOM.withUnit && in.Unit != nil && !strings.HasSuffix(compliantName, "_"+*in.Unit) { - compliantName = compliantName + "_" + *in.Unit - } // Comments, first HELP, then TYPE. if in.Help != nil { @@ -217,7 +193,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...E if err != nil { return written, err } - if toOM.withUnit && in.Unit != nil { + if in.Unit != nil { n, err = w.WriteString("# UNIT ") written += n if err != nil { diff --git a/vendor/github.com/prometheus/common/expfmt/text_create.go b/vendor/github.com/prometheus/common/expfmt/text_create.go index 6b8978145..f4074ae9a 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_create.go +++ b/vendor/github.com/prometheus/common/expfmt/text_create.go @@ -42,12 +42,12 @@ const ( var ( bufPool = sync.Pool{ - New: func() interface{} { + New: func() any { return bufio.NewWriter(io.Discard) }, } numBufPool = sync.Pool{ - New: func() interface{} { + New: func() any { b := make([]byte, 0, initialNumBufSize) return &b }, diff --git a/vendor/github.com/prometheus/common/expfmt/text_parse.go b/vendor/github.com/prometheus/common/expfmt/text_parse.go index 00c8841a1..4ce1f40b8 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_parse.go +++ b/vendor/github.com/prometheus/common/expfmt/text_parse.go @@ -339,6 +339,16 @@ func (p *TextParser) startLabelName() stateFn { return nil // Unexpected end of input. } if p.currentByte == '}' { + if p.currentMF == nil { + // The closing brace was reached before any metric name was read, + // e.g. for the input "{}". There is no metric to attach labels to, + // so this is a malformed exposition. This mirrors the guard in + // startLabelValue. currentMF (not currentMetric) is checked because + // reset only clears currentMF between parses. + p.parseError("invalid metric name") + p.currentLabelPairs = nil + return nil + } p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPairs...) p.currentLabelPairs = nil if p.skipBlankTab(); p.err != nil { diff --git a/vendor/github.com/prometheus/common/model/labels.go b/vendor/github.com/prometheus/common/model/labels.go index dfeb34be5..29688a13c 100644 --- a/vendor/github.com/prometheus/common/model/labels.go +++ b/vendor/github.com/prometheus/common/model/labels.go @@ -124,7 +124,7 @@ func (ln LabelName) IsValidLegacy() bool { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (ln *LabelName) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (ln *LabelName) UnmarshalYAML(unmarshal func(any) error) error { var s string if err := unmarshal(&s); err != nil { return err diff --git a/vendor/github.com/prometheus/common/model/labelset.go b/vendor/github.com/prometheus/common/model/labelset.go index 9de47b256..6010b26a8 100644 --- a/vendor/github.com/prometheus/common/model/labelset.go +++ b/vendor/github.com/prometheus/common/model/labelset.go @@ -16,6 +16,7 @@ package model import ( "encoding/json" "fmt" + "maps" "sort" ) @@ -107,9 +108,7 @@ func (ls LabelSet) Before(o LabelSet) bool { // Clone returns a copy of the label set. func (ls LabelSet) Clone() LabelSet { lsn := make(LabelSet, len(ls)) - for ln, lv := range ls { - lsn[ln] = lv - } + maps.Copy(lsn, ls) return lsn } @@ -117,13 +116,9 @@ func (ls LabelSet) Clone() LabelSet { func (ls LabelSet) Merge(other LabelSet) LabelSet { result := make(LabelSet, len(ls)) - for k, v := range ls { - result[k] = v - } + maps.Copy(result, ls) - for k, v := range other { - result[k] = v - } + maps.Copy(result, other) return result } diff --git a/vendor/github.com/prometheus/common/model/metric.go b/vendor/github.com/prometheus/common/model/metric.go index 3feebf328..2fe461511 100644 --- a/vendor/github.com/prometheus/common/model/metric.go +++ b/vendor/github.com/prometheus/common/model/metric.go @@ -17,6 +17,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "regexp" "sort" "strconv" @@ -24,7 +25,6 @@ import ( "unicode/utf8" dto "github.com/prometheus/client_model/go" - "go.yaml.in/yaml/v2" "google.golang.org/protobuf/proto" ) @@ -78,14 +78,6 @@ const ( UTF8Validation ) -var _ interface { - yaml.Marshaler - yaml.Unmarshaler - json.Marshaler - json.Unmarshaler - fmt.Stringer -} = new(ValidationScheme) - // String returns the string representation of s. func (s ValidationScheme) String() string { switch s { @@ -267,9 +259,7 @@ func (m Metric) Before(o Metric) bool { // Clone returns a copy of the Metric. func (m Metric) Clone() Metric { clone := make(Metric, len(m)) - for k, v := range m { - clone[k] = v - } + maps.Copy(clone, m) return clone } diff --git a/vendor/github.com/prometheus/common/model/time.go b/vendor/github.com/prometheus/common/model/time.go index 1730b0fdc..0854753f4 100644 --- a/vendor/github.com/prometheus/common/model/time.go +++ b/vendor/github.com/prometheus/common/model/time.go @@ -123,44 +123,38 @@ func (t Time) MarshalJSON() ([]byte, error) { // UnmarshalJSON implements the json.Unmarshaler interface. func (t *Time) UnmarshalJSON(b []byte) error { - p := strings.Split(string(b), ".") - switch len(p) { - case 1: - v, err := strconv.ParseInt(p[0], 10, 64) + base, frac, found := strings.Cut(string(b), ".") + if !found { + v, err := strconv.ParseInt(base, 10, 64) if err != nil { return err } *t = Time(v * second) - - case 2: - v, err := strconv.ParseInt(p[0], 10, 64) + } else { + v, err := strconv.ParseInt(base, 10, 64) if err != nil { return err } - v *= second - prec := dotPrecision - len(p[1]) + prec := dotPrecision - len(frac) if prec < 0 { - p[1] = p[1][:dotPrecision] - } else if prec > 0 { - p[1] += strings.Repeat("0", prec) + frac = frac[:dotPrecision] } - - va, err := strconv.ParseInt(p[1], 10, 32) + va, err := strconv.ParseInt(frac, 10, 32) if err != nil { return err } - - // If the value was something like -0.1 the negative is lost in the - // parsing because of the leading zero, this ensures that we capture it. - if len(p[0]) > 0 && p[0][0] == '-' && v+va > 0 { - *t = Time(v+va) * -1 - } else { - *t = Time(v + va) + switch prec { + case 1: + va *= 10 + case 2: + va *= 100 } - default: - return fmt.Errorf("invalid time %q", string(b)) + if len(base) > 0 && base[0] == '-' { + va = -va + } + *t = Time(v*second + va) } return nil } @@ -340,12 +334,12 @@ func (d *Duration) UnmarshalText(text []byte) error { } // MarshalYAML implements the yaml.Marshaler interface. -func (d Duration) MarshalYAML() (interface{}, error) { +func (d Duration) MarshalYAML() (any, error) { return d.String(), nil } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (d *Duration) UnmarshalYAML(unmarshal func(any) error) error { var s string if err := unmarshal(&s); err != nil { return err diff --git a/vendor/github.com/prometheus/common/model/value.go b/vendor/github.com/prometheus/common/model/value.go index a9995a37e..8dffd9c4a 100644 --- a/vendor/github.com/prometheus/common/model/value.go +++ b/vendor/github.com/prometheus/common/model/value.go @@ -259,13 +259,13 @@ func (s Scalar) String() string { // MarshalJSON implements json.Marshaler. func (s Scalar) MarshalJSON() ([]byte, error) { v := strconv.FormatFloat(float64(s.Value), 'f', -1, 64) - return json.Marshal([...]interface{}{s.Timestamp, v}) + return json.Marshal([...]any{s.Timestamp, v}) } // UnmarshalJSON implements json.Unmarshaler. func (s *Scalar) UnmarshalJSON(b []byte) error { var f string - v := [...]interface{}{&s.Timestamp, &f} + v := [...]any{&s.Timestamp, &f} if err := json.Unmarshal(b, &v); err != nil { return err @@ -291,12 +291,12 @@ func (s *String) String() string { // MarshalJSON implements json.Marshaler. func (s String) MarshalJSON() ([]byte, error) { - return json.Marshal([]interface{}{s.Timestamp, s.Value}) + return json.Marshal([]any{s.Timestamp, s.Value}) } // UnmarshalJSON implements json.Unmarshaler. func (s *String) UnmarshalJSON(b []byte) error { - v := [...]interface{}{&s.Timestamp, &s.Value} + v := [...]any{&s.Timestamp, &s.Value} return json.Unmarshal(b, &v) } diff --git a/vendor/github.com/prometheus/common/model/value_float.go b/vendor/github.com/prometheus/common/model/value_float.go index 6bfc757d1..b7d93615e 100644 --- a/vendor/github.com/prometheus/common/model/value_float.go +++ b/vendor/github.com/prometheus/common/model/value_float.go @@ -79,7 +79,7 @@ func (s SamplePair) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil + return fmt.Appendf(nil, "[%s,%s]", t, v), nil } // UnmarshalJSON implements json.Unmarshaler. diff --git a/vendor/github.com/prometheus/common/model/value_histogram.go b/vendor/github.com/prometheus/common/model/value_histogram.go index 91ce5b7a4..f27856ccc 100644 --- a/vendor/github.com/prometheus/common/model/value_histogram.go +++ b/vendor/github.com/prometheus/common/model/value_histogram.go @@ -67,11 +67,11 @@ func (s HistogramBucket) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - return []byte(fmt.Sprintf("[%s,%s,%s,%s]", b, l, u, c)), nil + return fmt.Appendf(nil, "[%s,%s,%s,%s]", b, l, u, c), nil } func (s *HistogramBucket) UnmarshalJSON(buf []byte) error { - tmp := []interface{}{&s.Boundaries, &s.Lower, &s.Upper, &s.Count} + tmp := []any{&s.Boundaries, &s.Lower, &s.Upper, &s.Count} wantLen := len(tmp) if err := json.Unmarshal(buf, &tmp); err != nil { return err @@ -152,11 +152,11 @@ func (s SampleHistogramPair) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil + return fmt.Appendf(nil, "[%s,%s]", t, v), nil } func (s *SampleHistogramPair) UnmarshalJSON(buf []byte) error { - tmp := []interface{}{&s.Timestamp, &s.Histogram} + tmp := []any{&s.Timestamp, &s.Histogram} wantLen := len(tmp) if err := json.Unmarshal(buf, &tmp); err != nil { return err diff --git a/vendor/github.com/prometheus/procfs/Makefile.common b/vendor/github.com/prometheus/procfs/Makefile.common index cce3ef1d1..a7c5f553e 100644 --- a/vendor/github.com/prometheus/procfs/Makefile.common +++ b/vendor/github.com/prometheus/procfs/Makefile.common @@ -55,13 +55,13 @@ ifneq ($(shell command -v gotestsum 2> /dev/null),) endif endif -PROMU_VERSION ?= 0.18.0 +PROMU_VERSION ?= 0.20.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v2.10.1 +GOLANGCI_LINT_VERSION ?= v2.11.4 GOLANGCI_FMT_OPTS ?= # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. @@ -90,8 +90,8 @@ ifdef DOCKERFILE_PATH $(error DOCKERFILE_PATH is deprecated. Use DOCKERFILE_VARIANTS ?= $(DOCKERFILE_PATH) in the Makefile) endif -DOCKER_ARCHS ?= amd64 -DOCKERFILE_VARIANTS ?= Dockerfile $(wildcard Dockerfile.*) +DOCKER_ARCHS ?= amd64 arm64 armv7 ppc64le riscv64 s390x +DOCKERFILE_VARIANTS ?= $(wildcard Dockerfile Dockerfile.*) # Function to extract variant from Dockerfile label. # Returns the variant name from io.prometheus.image.variant label, or "default" if not found. @@ -109,24 +109,6 @@ endif # Build variant:dockerfile pairs for shell iteration. DOCKERFILE_VARIANTS_WITH_NAMES := $(foreach df,$(DOCKERFILE_VARIANTS),$(call dockerfile_variant,$(df)):$(df)) -# Shell helper to check whether a dockerfile/arch pair is excluded. -define dockerfile_arch_is_excluded -case " $(DOCKERFILE_ARCH_EXCLUSIONS) " in \ - *" $$dockerfile:$(1) "*) true ;; \ - *) false ;; \ -esac -endef - -# Shell helper to check whether a registry/arch pair is excluded. -# Extracts registry from DOCKER_REPO (e.g., quay.io/prometheus -> quay.io) -define registry_arch_is_excluded -registry=$$(echo "$(DOCKER_REPO)" | cut -d'/' -f1); \ -case " $(DOCKER_REGISTRY_ARCH_EXCLUSIONS) " in \ - *" $$registry:$(1) "*) true ;; \ - *) false ;; \ -esac -endef - BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS)) @@ -268,10 +250,6 @@ $(BUILD_DOCKER_ARCHS): common-docker-%: @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ dockerfile=$${variant#*:}; \ variant_name=$${variant%%:*}; \ - if $(call dockerfile_arch_is_excluded,$*); then \ - echo "Skipping $$variant_name variant for linux-$* (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ distroless_arch="$*"; \ if [ "$*" = "armv7" ]; then \ distroless_arch="arm"; \ @@ -306,14 +284,6 @@ $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ dockerfile=$${variant#*:}; \ variant_name=$${variant%%:*}; \ - if $(call dockerfile_arch_is_excluded,$*); then \ - echo "Skipping push for $$variant_name variant on linux-$* (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$*); then \ - echo "Skipping push for $$variant_name variant on linux-$* to $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ echo "Pushing $$variant_name variant for linux-$*"; \ docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ @@ -341,14 +311,6 @@ $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: @for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \ dockerfile=$${variant#*:}; \ variant_name=$${variant%%:*}; \ - if $(call dockerfile_arch_is_excluded,$*); then \ - echo "Skipping tag for $$variant_name variant on linux-$* (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$*); then \ - echo "Skipping tag for $$variant_name variant on linux-$* for $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ if [ "$$dockerfile" != "Dockerfile" ] || [ "$$variant_name" != "default" ]; then \ echo "Tagging $$variant_name variant for linux-$* as latest"; \ docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest-$$variant_name"; \ @@ -370,14 +332,6 @@ common-docker-manifest: echo "Creating manifest for $$variant_name variant"; \ refs=""; \ for arch in $(DOCKER_ARCHS); do \ - if $(call dockerfile_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for $$variant_name (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for $$variant_name on $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name"; \ done; \ if [ -z "$$refs" ]; then \ @@ -391,14 +345,6 @@ common-docker-manifest: echo "Creating default variant ($$variant_name) manifest"; \ refs=""; \ for arch in $(DOCKER_ARCHS); do \ - if $(call dockerfile_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for default variant (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for default variant on $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:$(SANITIZED_DOCKER_IMAGE_TAG)"; \ done; \ if [ -z "$$refs" ]; then \ @@ -413,14 +359,6 @@ common-docker-manifest: echo "Creating manifest for $$variant_name variant version tag"; \ refs=""; \ for arch in $(DOCKER_ARCHS); do \ - if $(call dockerfile_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for $$variant_name version tag (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for $$variant_name version tag on $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:v$(DOCKER_MAJOR_VERSION_TAG)-$$variant_name"; \ done; \ if [ -z "$$refs" ]; then \ @@ -434,14 +372,6 @@ common-docker-manifest: echo "Creating default variant version tag manifest"; \ refs=""; \ for arch in $(DOCKER_ARCHS); do \ - if $(call dockerfile_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for default variant version tag (excluded by DOCKERFILE_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ - if $(call registry_arch_is_excluded,$$arch); then \ - echo " Skipping $$arch for default variant version tag on $(DOCKER_REPO) (excluded by DOCKER_REGISTRY_ARCH_EXCLUSIONS)"; \ - continue; \ - fi; \ refs="$$refs $(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$$arch:v$(DOCKER_MAJOR_VERSION_TAG)"; \ done; \ if [ -z "$$refs" ]; then \ @@ -495,9 +425,3 @@ $(1)_precheck: exit 1; \ fi endef - -govulncheck: install-govulncheck - govulncheck ./... - -install-govulncheck: - command -v govulncheck > /dev/null || go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/vendor/github.com/prometheus/procfs/README.md b/vendor/github.com/prometheus/procfs/README.md index 0718239cf..363524094 100644 --- a/vendor/github.com/prometheus/procfs/README.md +++ b/vendor/github.com/prometheus/procfs/README.md @@ -7,7 +7,7 @@ metrics from the pseudo-filesystems /proc and /sys. backwards-incompatible ways without warnings. Use it at your own risk. [![Go Reference](https://pkg.go.dev/badge/github.com/prometheus/procfs.svg)](https://pkg.go.dev/github.com/prometheus/procfs) -[![CircleCI](https://circleci.com/gh/prometheus/procfs/tree/master.svg?style=svg)](https://circleci.com/gh/prometheus/procfs/tree/master) +[![Build Status](https://github.com/prometheus/procfs/actions/workflows/ci.yml/badge.svg)](https://github.com/prometheus/procfs/actions/workflows/ci.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/procfs)](https://goreportcard.com/report/github.com/prometheus/procfs) ## Usage diff --git a/vendor/github.com/prometheus/procfs/SECURITY.md b/vendor/github.com/prometheus/procfs/SECURITY.md index fed02d85c..5e6f976db 100644 --- a/vendor/github.com/prometheus/procfs/SECURITY.md +++ b/vendor/github.com/prometheus/procfs/SECURITY.md @@ -3,4 +3,4 @@ The Prometheus security policy, including how to report vulnerabilities, can be found here: - +[https://prometheus.io/docs/operating/security/](https://prometheus.io/docs/operating/security/) diff --git a/vendor/github.com/prometheus/procfs/crypto.go b/vendor/github.com/prometheus/procfs/crypto.go index e4a5876ea..d93b712e0 100644 --- a/vendor/github.com/prometheus/procfs/crypto.go +++ b/vendor/github.com/prometheus/procfs/crypto.go @@ -48,11 +48,13 @@ type Crypto struct { Walksize *uint64 } +var cryptoFile = "crypto" + // Crypto parses an crypto-file (/proc/crypto) and returns a slice of // structs containing the relevant info. More information available here: // https://kernel.readthedocs.io/en/sphinx-samples/crypto-API.html func (fs FS) Crypto() ([]Crypto, error) { - path := fs.proc.Path("crypto") + path := fs.proc.Path(cryptoFile) b, err := util.ReadFileNoStat(path) if err != nil { return nil, fmt.Errorf("%w: Cannot read file %v: %w", ErrFileRead, b, err) @@ -82,6 +84,10 @@ func parseCrypto(r io.Reader) ([]Crypto, error) { continue } + if len(out) == 0 { + return nil, fmt.Errorf("%w: parsed invalid line before name parsed: %q", ErrFileParse, text) + } + kv := strings.Split(text, ":") if len(kv) != 2 { return nil, fmt.Errorf("%w: Cannot parse line: %q", ErrFileParse, text) diff --git a/vendor/github.com/prometheus/procfs/mountinfo.go b/vendor/github.com/prometheus/procfs/mountinfo.go index 9414a12f4..8594ae7f1 100644 --- a/vendor/github.com/prometheus/procfs/mountinfo.go +++ b/vendor/github.com/prometheus/procfs/mountinfo.go @@ -17,10 +17,10 @@ import ( "bufio" "bytes" "fmt" + "io" + "os" "strconv" "strings" - - "github.com/prometheus/procfs/internal/util" ) // A MountInfo is a type that describes the details, options @@ -160,9 +160,19 @@ func mountOptionsParser(mountOptions string) map[string]string { return opts } +// readMountInfo reads a full mountinfo file (no 1 MiB cap, unlike util.ReadFileNoStat). +func readMountInfo(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(f) +} + // GetMounts retrieves mountinfo information from `/proc/self/mountinfo`. func GetMounts() ([]*MountInfo, error) { - data, err := util.ReadFileNoStat("/proc/self/mountinfo") + data, err := readMountInfo("/proc/self/mountinfo") if err != nil { return nil, err } @@ -171,7 +181,7 @@ func GetMounts() ([]*MountInfo, error) { // GetProcMounts retrieves mountinfo information from a processes' `/proc//mountinfo`. func GetProcMounts(pid int) ([]*MountInfo, error) { - data, err := util.ReadFileNoStat(fmt.Sprintf("/proc/%d/mountinfo", pid)) + data, err := readMountInfo(fmt.Sprintf("/proc/%d/mountinfo", pid)) if err != nil { return nil, err } @@ -180,7 +190,7 @@ func GetProcMounts(pid int) ([]*MountInfo, error) { // GetMounts retrieves mountinfo information from `/proc/self/mountinfo`. func (fs FS) GetMounts() ([]*MountInfo, error) { - data, err := util.ReadFileNoStat(fs.proc.Path("self/mountinfo")) + data, err := readMountInfo(fs.proc.Path("self/mountinfo")) if err != nil { return nil, err } @@ -189,7 +199,7 @@ func (fs FS) GetMounts() ([]*MountInfo, error) { // GetProcMounts retrieves mountinfo information from a processes' `/proc//mountinfo`. func (fs FS) GetProcMounts(pid int) ([]*MountInfo, error) { - data, err := util.ReadFileNoStat(fs.proc.Path(fmt.Sprintf("%d/mountinfo", pid))) + data, err := readMountInfo(fs.proc.Path(fmt.Sprintf("%d/mountinfo", pid))) if err != nil { return nil, err } diff --git a/vendor/github.com/prometheus/procfs/net_wireless.go b/vendor/github.com/prometheus/procfs/net_wireless.go index 69d079445..f74dd3bed 100644 --- a/vendor/github.com/prometheus/procfs/net_wireless.go +++ b/vendor/github.com/prometheus/procfs/net_wireless.go @@ -114,47 +114,47 @@ func parseWireless(r io.Reader) ([]*Wireless, error) { qlink, err := strconv.Atoi(strings.TrimSuffix(stats[1], ".")) if err != nil { - return nil, fmt.Errorf("%w: parse Quality:link as integer %q: %w", ErrFileParse, qlink, err) + return nil, fmt.Errorf("%w: parse Quality:link as integer %q: %w", ErrFileParse, stats[1], err) } qlevel, err := strconv.Atoi(strings.TrimSuffix(stats[2], ".")) if err != nil { - return nil, fmt.Errorf("%w: Quality:level as integer %q: %w", ErrFileParse, qlevel, err) + return nil, fmt.Errorf("%w: Quality:level as integer %q: %w", ErrFileParse, stats[2], err) } qnoise, err := strconv.Atoi(strings.TrimSuffix(stats[3], ".")) if err != nil { - return nil, fmt.Errorf("%w: Quality:noise as integer %q: %w", ErrFileParse, qnoise, err) + return nil, fmt.Errorf("%w: Quality:noise as integer %q: %w", ErrFileParse, stats[3], err) } dnwid, err := strconv.Atoi(stats[4]) if err != nil { - return nil, fmt.Errorf("%w: Discarded:nwid as integer %q: %w", ErrFileParse, dnwid, err) + return nil, fmt.Errorf("%w: Discarded:nwid as integer %q: %w", ErrFileParse, stats[4], err) } dcrypt, err := strconv.Atoi(stats[5]) if err != nil { - return nil, fmt.Errorf("%w: Discarded:crypt as integer %q: %w", ErrFileParse, dcrypt, err) + return nil, fmt.Errorf("%w: Discarded:crypt as integer %q: %w", ErrFileParse, stats[5], err) } dfrag, err := strconv.Atoi(stats[6]) if err != nil { - return nil, fmt.Errorf("%w: Discarded:frag as integer %q: %w", ErrFileParse, dfrag, err) + return nil, fmt.Errorf("%w: Discarded:frag as integer %q: %w", ErrFileParse, stats[6], err) } dretry, err := strconv.Atoi(stats[7]) if err != nil { - return nil, fmt.Errorf("%w: Discarded:retry as integer %q: %w", ErrFileParse, dretry, err) + return nil, fmt.Errorf("%w: Discarded:retry as integer %q: %w", ErrFileParse, stats[7], err) } dmisc, err := strconv.Atoi(stats[8]) if err != nil { - return nil, fmt.Errorf("%w: Discarded:misc as integer %q: %w", ErrFileParse, dmisc, err) + return nil, fmt.Errorf("%w: Discarded:misc as integer %q: %w", ErrFileParse, stats[8], err) } mbeacon, err := strconv.Atoi(stats[9]) if err != nil { - return nil, fmt.Errorf("%w: Missed:beacon as integer %q: %w", ErrFileParse, mbeacon, err) + return nil, fmt.Errorf("%w: Missed:beacon as integer %q: %w", ErrFileParse, stats[9], err) } w := &Wireless{ diff --git a/vendor/github.com/prometheus/procfs/proc_cgroup.go b/vendor/github.com/prometheus/procfs/proc_cgroup.go index 535c08d6f..7e8a12297 100644 --- a/vendor/github.com/prometheus/procfs/proc_cgroup.go +++ b/vendor/github.com/prometheus/procfs/proc_cgroup.go @@ -60,7 +60,7 @@ func parseCgroupString(cgroupStr string) (*Cgroup, error) { } cgroup.HierarchyID, err = strconv.Atoi(fields[0]) if err != nil { - return nil, fmt.Errorf("%w: hierarchy ID: %q", ErrFileParse, cgroup.HierarchyID) + return nil, fmt.Errorf("%w: hierarchy ID: %q", ErrFileParse, fields[0]) } if fields[1] != "" { ssNames := strings.Split(fields[1], ",") diff --git a/vendor/github.com/schollz/progressbar/v3/progressbar.go b/vendor/github.com/schollz/progressbar/v3/progressbar.go index 99222a3c7..37158617b 100644 --- a/vendor/github.com/schollz/progressbar/v3/progressbar.go +++ b/vendor/github.com/schollz/progressbar/v3/progressbar.go @@ -81,6 +81,8 @@ type config struct { colorCodes bool // custom colors to use for colorCodes customColors map[string]string + // color to apply to spinner + spinnerColorCode string // show rate of change in kB/sec or MB/sec showBytes bool @@ -290,6 +292,13 @@ func OptionSetCustomColorCodes(customColors map[string]string) Option { } } +// OptionSetSpinnerColorCode sets color code for spinner +func OptionSetSpinnerColorCode(colorCode string) Option { + return func(p *ProgressBar) { + p.config.spinnerColorCode = colorCode + } +} + // OptionSetElapsedTime will enable elapsed time. Always enabled if OptionSetPredictTime is true. func OptionSetElapsedTime(elapsedTime bool) Option { return func(p *ProgressBar) { @@ -1114,6 +1123,74 @@ func getStringWidth(c config, str string) int { return stringWidth } +func renderDeterminateProgressBar(c config, s *state, bar, stats, leftBrac, rightBrac string) string { + if rightBrac == "" { + str := fmt.Sprintf("%4d%% %s %s", + s.currentPercent, + bar, + stats) + if (s.currentPercent == 100 && c.showElapsedTimeOnFinish) || c.elapsedTime { + str = fmt.Sprintf("%s [%s]", str, leftBrac) + } + + if c.showDescriptionAtLineEnd { + return fmt.Sprintf("\r%s %s ", str, c.description) + } + return fmt.Sprintf("\r%s%s ", c.description, str) + } + + if s.currentPercent == 100 { + str := fmt.Sprintf("%4d%% %s %s", + s.currentPercent, + bar, + stats) + + if c.showElapsedTimeOnFinish { + str = fmt.Sprintf("%s [%s]", str, leftBrac) + } + + if c.showDescriptionAtLineEnd { + return fmt.Sprintf("\r%s %s", str, c.description) + } + return fmt.Sprintf("\r%s%s", c.description, str) + } + + str := fmt.Sprintf("%4d%% %s %s [%s:%s]", + s.currentPercent, + bar, + stats, + leftBrac, + rightBrac) + + if c.showDescriptionAtLineEnd { + return fmt.Sprintf("\r%s %s", str, c.description) + } + return fmt.Sprintf("\r%s%s", c.description, str) +} + +func fitProgressBarWidth(c config, s *state, barStart, barEnd, stats, leftBrac, rightBrac string) int { + if c.fullWidth || c.ignoreLength || c.width <= 0 { + return c.width + } + + terminalWidth, err := termWidth(c.writer) + if err != nil || terminalWidth <= 0 { + return c.width + } + + bar := barStart + strings.Repeat(c.theme.SaucerPadding, c.width) + barEnd + lineWidth := getStringWidth(c, renderDeterminateProgressBar(c, s, bar, stats, leftBrac, rightBrac)) + if lineWidth <= terminalWidth { + return c.width + } + + fitWidth := c.width - (lineWidth - terminalWidth) + if fitWidth < 0 { + return 0 + } + return fitWidth +} + func renderProgressBar(c config, s *state) (int, error) { var sb strings.Builder @@ -1243,6 +1320,10 @@ func renderProgressBar(c config, s *state) (int, error) { if (s.currentSaucerSize > 0 || s.currentPercent > 0) && c.theme.BarStartFilled != "" { barStart = c.theme.BarStartFilled } + c.width = fitProgressBarWidth(c, s, barStart, barEnd, sb.String(), leftBrac, rightBrac) + if !c.ignoreLength { + s.currentSaucerSize = int(float64(s.currentPercent) / 100.0 * float64(c.width)) + } if s.currentSaucerSize > 0 { if c.ignoreLength { saucer = strings.Repeat(c.theme.SaucerPadding, s.currentSaucerSize-1) @@ -1294,6 +1375,10 @@ func renderProgressBar(c config, s *state) (int, error) { spinner = selectedSpinner[s.spinnerIdx] s.spinnerIdx = (s.spinnerIdx + 1) % len(selectedSpinner) } + // if set add spinner color code + if c.spinnerColorCode != "" && c.colorCodes { + spinner = "[" + c.spinnerColorCode + "]" + spinner + "[reset]" + } if c.elapsedTime { if c.showDescriptionAtLineEnd { str = fmt.Sprintf("\r%s %s [%s] %s ", @@ -1322,61 +1407,11 @@ func renderProgressBar(c config, s *state) (int, error) { } } } else if rightBrac == "" { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s", - s.currentPercent, - barStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - barEnd, - sb.String()) - if (s.currentPercent == 100 && c.showElapsedTimeOnFinish) || c.elapsedTime { - str = fmt.Sprintf("%s [%s]", str, leftBrac) - } - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s ", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s ", c.description, str) - } + bar := barStart + saucer + saucerHead + strings.Repeat(c.theme.SaucerPadding, repeatAmount) + barEnd + str = renderDeterminateProgressBar(c, s, bar, sb.String(), leftBrac, rightBrac) } else { - if s.currentPercent == 100 { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s", - s.currentPercent, - barStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - barEnd, - sb.String()) - - if c.showElapsedTimeOnFinish { - str = fmt.Sprintf("%s [%s]", str, leftBrac) - } - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s", c.description, str) - } - } else { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s [%s:%s]", - s.currentPercent, - barStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - barEnd, - sb.String(), - leftBrac, - rightBrac) - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s", c.description, str) - } - } + bar := barStart + saucer + saucerHead + strings.Repeat(c.theme.SaucerPadding, repeatAmount) + barEnd + str = renderDeterminateProgressBar(c, s, bar, sb.String(), leftBrac, rightBrac) } if c.colorCodes { diff --git a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1alpha1/build_types.go b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1alpha1/build_types.go index 7ccee7517..be03259f3 100644 --- a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1alpha1/build_types.go +++ b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1alpha1/build_types.go @@ -241,11 +241,13 @@ type Image struct { // Annotations references the additional annotations to be applied on the image // // +optional + // +kubebuilder:validation:XValidation:rule="self.all(k, k != '' && !k.contains('='))",message="annotation keys must not be empty and must not contain '='" Annotations map[string]string `json:"annotations,omitempty"` // Labels references the additional labels to be applied on the image // // +optional + // +kubebuilder:validation:XValidation:rule="self.all(k, k != '' && !k.contains('='))",message="label keys must not be empty and must not contain '='" Labels map[string]string `json:"labels,omitempty"` // Timestamp references the optional image timestamp to be set, valid values are: diff --git a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/build_types.go b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/build_types.go index 9868b43a5..1bd530a9a 100644 --- a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/build_types.go +++ b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/build_types.go @@ -37,6 +37,8 @@ const ( SpecEnvNameCanNotBeBlank BuildReason = "SpecEnvNameCanNotBeBlank" // SpecEnvOnlyOneOfValueOrValueFromMustBeSpecified indicates that both value and valueFrom were specified SpecEnvOnlyOneOfValueOrValueFromMustBeSpecified BuildReason = "SpecEnvOnlyOneOfValueOrValueFromMustBeSpecified" + // SpecEnvNameForbidden indicates that an environment variable name is on the security blocklist + SpecEnvNameForbidden BuildReason = "SpecEnvNameForbidden" // RuntimePathsCanNotBeEmpty indicates that the spec.runtime feature is used but the paths were not specified RuntimePathsCanNotBeEmpty BuildReason = "RuntimePathsCanNotBeEmpty" // RestrictedParametersInUse indicates the definition of reserved shipwright parameters @@ -312,11 +314,13 @@ type Image struct { // Annotations references the additional annotations to be applied on the image // // +optional + // +kubebuilder:validation:XValidation:rule="self.all(k, k != '' && !k.contains('='))",message="annotation keys must not be empty and must not contain '='" Annotations map[string]string `json:"annotations,omitempty"` // Labels references the additional labels to be applied on the image // // +optional + // +kubebuilder:validation:XValidation:rule="self.all(k, k != '' && !k.contains('='))",message="label keys must not be empty and must not contain '='" Labels map[string]string `json:"labels,omitempty"` // VulnerabilityScan provides configurations about running a scan for your generated image diff --git a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/buildrun_conversion.go b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/buildrun_conversion.go index 081aea69f..efbade549 100644 --- a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/buildrun_conversion.go +++ b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/buildrun_conversion.go @@ -81,8 +81,10 @@ func (src *BuildRun) ConvertTo(ctx context.Context, obj *unstructured.Unstructur if src.Spec.Output != nil { alphaBuildRun.Spec.Output = &buildapialpha.Image{ Image: src.Spec.Output.Image, + Insecure: src.Spec.Output.Insecure, Annotations: src.Spec.Output.Annotations, Labels: src.Spec.Output.Labels, + Timestamp: src.Spec.Output.Timestamp, } if src.Spec.Output.PushSecret != nil { alphaBuildRun.Spec.Output.Credentials = &corev1.LocalObjectReference{ @@ -348,8 +350,10 @@ func (dest *BuildRunSpec) ConvertFrom(ctx context.Context, orig *buildapialpha.B if orig.Output != nil { dest.Output = &Image{ Image: orig.Output.Image, + Insecure: orig.Output.Insecure, Annotations: orig.Output.Annotations, Labels: orig.Output.Labels, + Timestamp: orig.Output.Timestamp, } if orig.Output.Credentials != nil { diff --git a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/parameter.go b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/parameter.go index 283e67d15..95711705b 100644 --- a/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/parameter.go +++ b/vendor/github.com/shipwright-io/build/pkg/apis/build/v1beta1/parameter.go @@ -25,15 +25,15 @@ type SingleValue struct { // The value of the parameter // +optional - Value *string `json:"value"` + Value *string `json:"value,omitempty"` // The ConfigMap value of the parameter // +optional - ConfigMapValue *ObjectKeyRef `json:"configMapValue"` + ConfigMapValue *ObjectKeyRef `json:"configMapValue,omitempty"` // The secret value of the parameter // +optional - SecretValue *ObjectKeyRef `json:"secretValue"` + SecretValue *ObjectKeyRef `json:"secretValue,omitempty"` } // ParamValue is a key/value that populates a strategy parameter diff --git a/vendor/github.com/shipwright-io/build/pkg/config/config.go b/vendor/github.com/shipwright-io/build/pkg/config/config.go index 24a4e2fcc..78af9d200 100644 --- a/vendor/github.com/shipwright-io/build/pkg/config/config.go +++ b/vendor/github.com/shipwright-io/build/pkg/config/config.go @@ -14,6 +14,8 @@ import ( "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" + + "github.com/shipwright-io/build/pkg/env" ) const ( @@ -80,8 +82,37 @@ const ( // environment variable to hold vulnerability count limit VulnerabilityCountLimitEnvVar = "VULNERABILITY_COUNT_LIMIT" + + // environment variable to override the forbidden env var blocklist + forbiddenEnvVarNamesEnvVar = "FORBIDDEN_ENV_VAR_NAMES" ) +// defaultForbiddenEnvVarNames is the default blocklist of environment variable +// names that are rejected for security reasons. Entries ending with "*" are +// treated as prefix matches (e.g. "LD_*" blocks any variable starting with "LD_"). +var defaultForbiddenEnvVarNames = []string{ + "LD_*", + "BASH_FUNC_*", + "LD_PRELOAD", + "LD_LIBRARY_PATH", + "LD_AUDIT", + "LD_DEBUG", + "LD_PROFILE", + "BASH_ENV", + "ENV", + "CDPATH", + "PYTHONSTARTUP", + "PERL5OPT", + "PERLLIB", + "PERL5LIB", + "RUBYOPT", + "NODE_OPTIONS", +} + +func init() { + env.SetForbiddenEnvVars(defaultForbiddenEnvVarNames) +} + var ( // arrays are not possible as constants metricBuildRunCompletionDurationBuckets = prometheus.LinearBuckets(50, 50, 10) @@ -109,6 +140,7 @@ type Config struct { GitRewriteRule bool VulnerabilityCountLimit int BuildrunExecutor string + ForbiddenEnvVarNames []string } // PrometheusConfig contains the specific configuration for the @@ -166,6 +198,7 @@ func NewDefaultConfig() *Config { GitRewriteRule: false, VulnerabilityCountLimit: 50, BuildrunExecutor: "TaskRun", + ForbiddenEnvVarNames: defaultForbiddenEnvVarNames, GitContainerTemplate: Step{ Image: gitDefaultImage, @@ -468,6 +501,15 @@ func (c *Config) SetConfigFromEnv() error { c.TerminationLogPath = terminationLogPath } + if forbiddenNames := os.Getenv(forbiddenEnvVarNamesEnvVar); forbiddenNames != "" { + entries := strings.Split(forbiddenNames, ",") + for i, entry := range entries { + entries[i] = strings.TrimSpace(entry) + } + c.ForbiddenEnvVarNames = entries + } + env.SetForbiddenEnvVars(c.ForbiddenEnvVarNames) + return nil } diff --git a/vendor/github.com/shipwright-io/build/pkg/env/env.go b/vendor/github.com/shipwright-io/build/pkg/env/env.go new file mode 100644 index 000000000..6b28f8f0f --- /dev/null +++ b/vendor/github.com/shipwright-io/build/pkg/env/env.go @@ -0,0 +1,100 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package env + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" +) + +var ( + forbiddenEnvVarNames = map[string]bool{} + forbiddenEnvVarPrefixes []string +) + +// SetForbiddenEnvVars replaces the forbidden environment variable blocklist. +// Entries ending with "*" are treated as prefix matches; all others are exact matches. +func SetForbiddenEnvVars(entries []string) { + names := make(map[string]bool) + var prefixes []string + for _, e := range entries { + e = strings.TrimSpace(e) + if e == "" { + continue + } + if strings.HasSuffix(e, "*") { + prefixes = append(prefixes, strings.TrimSuffix(e, "*")) + } else { + names[e] = true + } + } + forbiddenEnvVarNames = names + forbiddenEnvVarPrefixes = prefixes +} + +func IsForbiddenEnvVar(name string) bool { + if forbiddenEnvVarNames[name] { + return true + } + for _, prefix := range forbiddenEnvVarPrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// MergeEnvVars merges one slice of corev1.EnvVar into another slice of corev1.EnvVar +// if overwriteValues is false, this function will return an error if a duplicate EnvVar name is encountered +// if overwriteValues is true, this function will overwrite the existing value with the new value if a duplicate is encountered +func MergeEnvVars(from []corev1.EnvVar, into []corev1.EnvVar, overwriteValues bool) ([]corev1.EnvVar, error) { + // if from, into, or both are empty, there is no need to run through the processing logic + // just quickly return the appropriate value + if len(from) == 0 && len(into) == 0 { + return []corev1.EnvVar{}, nil + } else if len(from) == 0 { + return into, nil + } else if len(into) == 0 { + return from, nil + } + + // create a map of the original (into) env vars with the name as the key and + // their index as the value so we can do value replacements later if overwriteValues is true + originalEnvs := make(map[string]int) + + for i, o := range into { + originalEnvs[o.Name] = i + } + + // errs holds a slice of error objects from the merge process + var errs []error + + // merge the new env vars into the original env vars list following a few simple rules + // based on if the name already exists and whether overwriteValues is true or false + for _, n := range from { + if IsForbiddenEnvVar(n.Name) { + errs = append(errs, fmt.Errorf("environment variable %q is forbidden for security reasons", n.Name)) + continue + } + + _, exists := originalEnvs[n.Name] + + switch { + case exists && overwriteValues: + into[originalEnvs[n.Name]] = n + case exists && !overwriteValues: + errs = append(errs, fmt.Errorf("environment variable %q already exists", n.Name)) + default: + into = append(into, n) + } + } + + // kerrors.NewAggregate will return nil if the slice is empty + // or an aggregated error otherwise + return into, kerrors.NewAggregate(errs) +} diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/default.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/default.go index d25ab8bec..7031a7c8f 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/default.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/default.go @@ -51,6 +51,8 @@ const ( DefaultImagePullBackOffTimeout = 0 * time.Minute + DefaultCreateContainerErrorTimeout = 0 * time.Minute + // Default maximum resolution timeout used by the resolution controller before timing out when exceeded DefaultMaximumResolutionTimeout = 1 * time.Minute @@ -71,6 +73,7 @@ const ( defaultResolverTypeKey = "default-resolver-type" defaultContainerResourceRequirementsKey = "default-container-resource-requirements" defaultImagePullBackOffTimeout = "default-imagepullbackoff-timeout" + defaultCreateContainerErrorTimeout = "default-create-container-error-timeout" defaultMaximumResolutionTimeout = "default-maximum-resolution-timeout" defaultSidecarLogPollingIntervalKey = "default-sidecar-log-polling-interval" DefaultStepRefConcurrencyLimitKey = "default-step-ref-concurrency-limit" @@ -94,6 +97,7 @@ type Defaults struct { DefaultResolverType string DefaultContainerResourceRequirements map[string]corev1.ResourceRequirements DefaultImagePullBackOffTimeout time.Duration + DefaultCreateContainerErrorTimeout time.Duration DefaultMaximumResolutionTimeout time.Duration // DefaultSidecarLogPollingInterval specifies how frequently (as a time.Duration) the Tekton sidecar log results container polls for step completion files. // This value is loaded from the 'sidecar-log-polling-interval' key in the config-defaults ConfigMap. @@ -131,6 +135,7 @@ func (cfg *Defaults) Equals(other *Defaults) bool { other.DefaultMaxMatrixCombinationsCount == cfg.DefaultMaxMatrixCombinationsCount && other.DefaultResolverType == cfg.DefaultResolverType && other.DefaultImagePullBackOffTimeout == cfg.DefaultImagePullBackOffTimeout && + other.DefaultCreateContainerErrorTimeout == cfg.DefaultCreateContainerErrorTimeout && other.DefaultMaximumResolutionTimeout == cfg.DefaultMaximumResolutionTimeout && other.DefaultSidecarLogPollingInterval == cfg.DefaultSidecarLogPollingInterval && other.DefaultStepRefConcurrencyLimit == cfg.DefaultStepRefConcurrencyLimit && @@ -140,16 +145,17 @@ func (cfg *Defaults) Equals(other *Defaults) bool { // NewDefaultsFromMap returns a Config given a map corresponding to a ConfigMap func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) { tc := Defaults{ - DefaultTimeoutMinutes: DefaultTimeoutMinutes, - DefaultServiceAccount: DefaultServiceAccountValue, - DefaultManagedByLabelValue: DefaultManagedByLabelValue, - DefaultCloudEventsSink: DefaultCloudEventSinkValue, - DefaultMaxMatrixCombinationsCount: DefaultMaxMatrixCombinationsCount, - DefaultResolverType: DefaultResolverTypeValue, - DefaultImagePullBackOffTimeout: DefaultImagePullBackOffTimeout, - DefaultMaximumResolutionTimeout: DefaultMaximumResolutionTimeout, - DefaultSidecarLogPollingInterval: DefaultSidecarLogPollingInterval, - DefaultStepRefConcurrencyLimit: DefaultStepRefConcurrencyLimit, + DefaultTimeoutMinutes: DefaultTimeoutMinutes, + DefaultServiceAccount: DefaultServiceAccountValue, + DefaultManagedByLabelValue: DefaultManagedByLabelValue, + DefaultCloudEventsSink: DefaultCloudEventSinkValue, + DefaultMaxMatrixCombinationsCount: DefaultMaxMatrixCombinationsCount, + DefaultResolverType: DefaultResolverTypeValue, + DefaultImagePullBackOffTimeout: DefaultImagePullBackOffTimeout, + DefaultCreateContainerErrorTimeout: DefaultCreateContainerErrorTimeout, + DefaultMaximumResolutionTimeout: DefaultMaximumResolutionTimeout, + DefaultSidecarLogPollingInterval: DefaultSidecarLogPollingInterval, + DefaultStepRefConcurrencyLimit: DefaultStepRefConcurrencyLimit, } if defaultTimeoutMin, ok := cfgMap[defaultTimeoutMinutesKey]; ok { @@ -228,6 +234,14 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) { tc.DefaultImagePullBackOffTimeout = timeout } + if defaultCreateContainerError, ok := cfgMap[defaultCreateContainerErrorTimeout]; ok { + timeout, err := time.ParseDuration(defaultCreateContainerError) + if err != nil { + return nil, fmt.Errorf("failed parsing default config %q: %w", defaultCreateContainerErrorTimeout, err) + } + tc.DefaultCreateContainerErrorTimeout = timeout + } + if defaultMaximumResolutionTimeout, ok := cfgMap[defaultMaximumResolutionTimeout]; ok { timeout, err := time.ParseDuration(defaultMaximumResolutionTimeout) if err != nil { diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/events.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/events.go index 0ae07b35e..acc9a99b5 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/events.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/events.go @@ -37,7 +37,7 @@ const ( ) var ( - // TODO(afrittoli): only one valid format for now, more to come + // validFormats is the set of supported event formats. // See TEP-0137 https://github.com/tektoncd/community/pull/1028 validFormats = EventFormats{FormatTektonV1: struct{}{}} diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/feature_flags.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/feature_flags.go index 07341c1b9..c2de63f06 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/config/feature_flags.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/config/feature_flags.go @@ -113,6 +113,14 @@ const ( EnableWaitExponentialBackoff = "enable-wait-exponential-backoff" // DefaultEnableWaitExponentialBackoff is the default value for EnableWaitExponentialBackoff DefaultEnableWaitExponentialBackoff = false + // EnableTerminationMessageCompression is the flag to enable compression of + // termination messages to fit more results in the 4KB Kubernetes limit. + // When enabled, results are compressed with flate and base64-encoded before + // writing to the termination message path. The reconciler auto-detects + // compressed vs plain JSON messages for backward compatibility. + EnableTerminationMessageCompression = "enable-termination-message-compression" + // DefaultEnableTerminationMessageCompression is the default value for EnableTerminationMessageCompression + DefaultEnableTerminationMessageCompression = false // EnableStepActions is the flag to enable step actions (no-op since it's stable) EnableStepActions = "enable-step-actions" @@ -178,6 +186,13 @@ var ( Enabled: DefaultAlphaFeatureEnabled, } + // DefaultEnableTerminationMessageCompressionFlag is the default PerFeatureFlag value for EnableTerminationMessageCompression + DefaultEnableTerminationMessageCompressionFlag = PerFeatureFlag{ + Name: EnableTerminationMessageCompression, + Stability: AlphaAPIFields, + Enabled: DefaultAlphaFeatureEnabled, + } + DefaultEnableTektonOCIBundles = PerFeatureFlag{ Name: EnableTektonOCIBundles, Stability: AlphaAPIFields, @@ -212,13 +227,14 @@ type FeatureFlags struct { Coschedule string `json:"coschedule,omitempty"` EnableCELInWhenExpression bool `json:"enableCELInWhenExpression,omitempty"` // EnableStepActions is a no-op flag since StepActions are stable - EnableStepActions bool `json:"enableStepActions,omitempty"` - EnableParamEnum bool `json:"enableParamEnum,omitempty"` - EnableArtifacts bool `json:"enableArtifacts,omitempty"` - DisableInlineSpec string `json:"disableInlineSpec,omitempty"` - EnableConciseResolverSyntax bool `json:"enableConciseResolverSyntax,omitempty"` - EnableKubernetesSidecar bool `json:"enableKubernetesSidecar,omitempty"` - EnableWaitExponentialBackoff bool `json:"enableWaitExponentialBackoff,omitempty"` + EnableStepActions bool `json:"enableStepActions,omitempty"` + EnableParamEnum bool `json:"enableParamEnum,omitempty"` + EnableArtifacts bool `json:"enableArtifacts,omitempty"` + DisableInlineSpec string `json:"disableInlineSpec,omitempty"` + EnableConciseResolverSyntax bool `json:"enableConciseResolverSyntax,omitempty"` + EnableKubernetesSidecar bool `json:"enableKubernetesSidecar,omitempty"` + EnableWaitExponentialBackoff bool `json:"enableWaitExponentialBackoff,omitempty"` + EnableTerminationMessageCompression bool `json:"enableTerminationMessageCompression,omitempty"` // DeprecatedEnableTektonOCIBundles is maintained for backward compatibility // to allow deletion of PipelineRuns created before v0.62.x. // This field is not used and can be removed in a future release @@ -332,6 +348,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) { if err := setFeature(EnableWaitExponentialBackoff, DefaultEnableWaitExponentialBackoff, &tc.EnableWaitExponentialBackoff); err != nil { return nil, err } + if err := setPerFeatureFlag(EnableTerminationMessageCompression, DefaultEnableTerminationMessageCompressionFlag, &tc.EnableTerminationMessageCompression); err != nil { + return nil, err + } return &tc, nil } diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/pod/doc.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/pod/doc.go index 9e7c8c335..501112522 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/pod/doc.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/pod/doc.go @@ -16,6 +16,6 @@ limitations under the License. // Package pod contains non-versioned pod configuration // +k8s:openapi-gen=true -// +gencrdrefdocs:unversionedTypes +// +versionName=unversioned // +groupName=tekton.dev package pod diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/matrix_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/matrix_types.go index 6de9aacd9..c58dd7b1c 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/matrix_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/matrix_types.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "maps" + "math" "sort" "github.com/tektoncd/pipeline/pkg/apis/config" @@ -221,7 +222,11 @@ func (m *Matrix) CountCombinations() int { count := m.countGeneratedCombinationsFromParams() // Add any additional Combinations generated from Matrix Include Parameters - count += m.countNewCombinationsFromInclude() + includeCount := m.countNewCombinationsFromInclude() + if count > math.MaxInt-includeCount { + return math.MaxInt + } + count += includeCount return count } @@ -234,8 +239,13 @@ func (m *Matrix) countGeneratedCombinationsFromParams() int { } count := 1 for _, param := range m.Params { - if len(param.Value.ArrayVal) > 0 { - count *= len(param.Value.ArrayVal) + if l := len(param.Value.ArrayVal); l > 0 { + // Cap at math.MaxInt on overflow so validateCombinationsCount + // still rejects the matrix instead of accepting a wrapped count. + if count > math.MaxInt/l { + return math.MaxInt + } + count *= l } } return count @@ -257,6 +267,9 @@ func (m *Matrix) countNewCombinationsFromInclude() int { if val, exist := matrixParamMap[param.Name]; exist { // If the Matrix Include param values does not exist, a new Combination will be generated if !slices.Contains(val, param.Value.StringVal) { + if count == math.MaxInt { + return math.MaxInt + } count++ } else { break diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/openapi_generated.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/openapi_generated.go index e59d64705..6cbcb68b5 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/openapi_generated.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/openapi_generated.go @@ -2027,13 +2027,13 @@ func schema_pkg_apis_pipeline_v1_PipelineTask(ref common.ReferenceCallback) comm }, "pipelineRef": { SchemaProps: spec.SchemaProps{ - Description: "PipelineRef is a reference to a pipeline definition Note: PipelineRef is in preview mode and not yet supported", + Description: "PipelineRef is a reference to a pipeline definition. This is an alpha field. You must set the \"enable-api-fields\" feature flag to \"alpha\" for this field to be supported. When enabled, the referenced Pipeline is executed as a child PipelineRun owned by the parent PipelineRun.", Ref: ref("github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.PipelineRef"), }, }, "pipelineSpec": { SchemaProps: spec.SchemaProps{ - Description: "PipelineSpec is a specification of a pipeline Note: PipelineSpec is in preview mode and not yet supported Specifying PipelineSpec can be disabled by setting `disable-inline-spec` feature flag. See Pipeline.spec (API version: tekton.dev/v1)", + Description: "PipelineSpec is a specification of a pipeline. This is an alpha field. You must set the \"enable-api-fields\" feature flag to \"alpha\" for this field to be supported. When enabled, the embedded Pipeline is executed as a child PipelineRun owned by the parent PipelineRun. Specifying PipelineSpec can be disabled by setting `disable-inline-spec` feature flag. See Pipeline.spec (API version: tekton.dev/v1)", Ref: ref("github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.PipelineSpec"), }, }, diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/param_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/param_types.go index 2d450941d..6906845c2 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/param_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/param_types.go @@ -32,6 +32,8 @@ import ( // ParamsPrefix is the prefix used in $(...) expressions referring to parameters const ParamsPrefix = "params" +const matrixField = "matrix" +const lengthField = "length" // ParamSpec defines arbitrary parameters needed beyond typed inputs (such as // resources). Parameter values are provided by users as inputs on a TaskRun @@ -260,12 +262,19 @@ func (p Param) ParseTaskandResultName() (string, string) { if expressions, ok := p.GetVarSubstitutionExpressions(); ok { for _, expression := range expressions { subExpressions := strings.Split(expression, ".") - pipelineTaskName := subExpressions[1] - if len(subExpressions) == 4 { - return pipelineTaskName, "" - } else if len(subExpressions) == 5 { - resultName := subExpressions[3] - return pipelineTaskName, resultName + // tasks..matrix.length + if len(subExpressions) == 4 && + subExpressions[0] == PipelineTasks && + subExpressions[2] == matrixField && + subExpressions[3] == lengthField { + return subExpressions[1], "" + } + // tasks..matrix..length + if len(subExpressions) == 5 && + subExpressions[0] == PipelineTasks && + subExpressions[2] == matrixField && + subExpressions[4] == lengthField { + return subExpressions[1], subExpressions[3] } } } diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_types.go index 4e7147f2b..0c44abf3c 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_types.go @@ -44,6 +44,7 @@ const ( ) // +genclient +// +kubebuilder:object:root=true // +genclient:noStatus // +genreconciler:krshapedlogic=false // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -242,13 +243,17 @@ type PipelineTask struct { // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` - // PipelineRef is a reference to a pipeline definition - // Note: PipelineRef is in preview mode and not yet supported + // PipelineRef is a reference to a pipeline definition. + // This is an alpha field. You must set the "enable-api-fields" feature flag + // to "alpha" for this field to be supported. When enabled, the referenced + // Pipeline is executed as a child PipelineRun owned by the parent PipelineRun. // +optional PipelineRef *PipelineRef `json:"pipelineRef,omitempty"` - // PipelineSpec is a specification of a pipeline - // Note: PipelineSpec is in preview mode and not yet supported + // PipelineSpec is a specification of a pipeline. + // This is an alpha field. You must set the "enable-api-fields" feature flag + // to "alpha" for this field to be supported. When enabled, the embedded + // Pipeline is executed as a child PipelineRun owned by the parent PipelineRun. // Specifying PipelineSpec can be disabled by setting // `disable-inline-spec` feature flag. // See Pipeline.spec (API version: tekton.dev/v1) diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_validation.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_validation.go index 0037d2973..f549ae3ed 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipeline_validation.go @@ -87,12 +87,18 @@ func (ps *PipelineSpec) Validate(ctx context.Context) (errs *apis.FieldError) { errs = errs.Also(validatePipelineWorkspacesDeclarations(ps.Workspaces)) // Validate the pipeline's results errs = errs.Also(validatePipelineResults(ps.Results, ps.Tasks, ps.Finally)) + // Reject result references to PipelineTasks that use pipelineRef or pipelineSpec. + // Result propagation from a child Pipeline is not implemented in the initial + // Pipelines-in-Pipelines alpha; see docs/pipelines-in-pipelines.md#limitations. + errs = errs.Also(validatePipelineRefResultReferencesDisallowed(ps.Tasks, ps.Finally)) errs = errs.Also(validateTasksAndFinallySection(ps)) errs = errs.Also(validateFinalTasks(ps.Tasks, ps.Finally)) errs = errs.Also(validateWhenExpressions(ctx, ps.Tasks, ps.Finally)) errs = errs.Also(validateArtifactReference(ctx, ps.Tasks, ps.Finally)) errs = errs.Also(validateMatrix(ctx, ps.Tasks).ViaField("tasks")) errs = errs.Also(validateMatrix(ctx, ps.Finally).ViaField("finally")) + errs = errs.Also(validateVarSubstitutionExpressions(ps.Tasks, "tasks")) + errs = errs.Also(validateVarSubstitutionExpressions(ps.Finally, "finally")) return errs } @@ -743,6 +749,43 @@ func taskContainsResult(resultExpression string, pipelineTaskNames sets.String, return true } +// validatePipelineRefResultReferencesDisallowed rejects any result reference +// (`$(tasks..results.*)` or `$(finally..results.*)`) whose target +// PipelineTask is a Pipeline-in-Pipeline (uses pipelineRef or pipelineSpec). +// +// Result propagation from a child Pipeline is not yet implemented: the +// reconciler's convertToResultRefs path reads from ResolvedPipelineTask.TaskRuns +// which is empty for a child-pipeline task, so without this guard such a +// reference would panic the controller at runtime. Once propagation lands, +// this guard can be removed. +func validatePipelineRefResultReferencesDisallowed(tasks []PipelineTask, finally []PipelineTask) (errs *apis.FieldError) { + pipelineRefTasks := sets.NewString() + for _, t := range tasks { + if t.PipelineRef != nil || t.PipelineSpec != nil { + pipelineRefTasks.Insert(t.Name) + } + } + if pipelineRefTasks.Len() == 0 { + return nil + } + + check := func(pts []PipelineTask, field string) { + for idx, pt := range pts { + for _, ref := range PipelineTaskResultRefs(&pt) { + if pipelineRefTasks.Has(ref.PipelineTask) { + errs = errs.Also(apis.ErrInvalidValue( + fmt.Sprintf("result reference to pipelineTask %q is not supported: referenced task uses pipelineRef or pipelineSpec and result propagation from child Pipelines is not yet implemented", + ref.PipelineTask), + "").ViaFieldIndex(field, idx)) + } + } + } + } + check(tasks, "tasks") + check(finally, "finally") + return errs +} + func validateTasksAndFinallySection(ps *PipelineSpec) *apis.FieldError { if len(ps.Finally) != 0 && len(ps.Tasks) == 0 { return apis.ErrInvalidValue(fmt.Sprintf("spec.tasks is empty but spec.finally has %d tasks", len(ps.Finally)), "finally") @@ -827,6 +870,69 @@ func validateMatrix(ctx context.Context, tasks []PipelineTask) (errs *apis.Field return errs } +func validateVarSubstitutionExpressions(tasks []PipelineTask, fieldPath string) (errs *apis.FieldError) { + validPrefixes := sets.NewString("params", "tasks", "finally", "context", "workspaces", "results") + for idx, task := range tasks { + for _, param := range task.Params { + if expressions, ok := param.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + prefix := strings.SplitN(expression, ".", 2)[0] + if !validPrefixes.Has(prefix) { + errs = errs.Also(apis.ErrInvalidValue( + fmt.Sprintf("invalid variable reference %q, must start with a valid prefix: params, tasks, finally, context, workspaces, or results; if you meant a shell variable, use ${VAR} instead", "$("+expression+")"), + "value", + ).ViaFieldKey("params", param.Name).ViaFieldIndex(fieldPath, idx)) + } + } + } + } + for i, we := range task.When { + if expressions, ok := we.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + prefix := strings.SplitN(expression, ".", 2)[0] + if !validPrefixes.Has(prefix) { + errs = errs.Also(apis.ErrInvalidValue( + fmt.Sprintf("invalid variable reference %q, must start with a valid prefix: params, tasks, finally, context, workspaces, or results; if you meant a shell variable, use ${VAR} instead", "$("+expression+")"), + "", + ).ViaFieldIndex("when", i).ViaFieldIndex(fieldPath, idx)) + } + } + } + } + if task.IsMatrixed() { + for _, param := range task.Matrix.Params { + if expressions, ok := param.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + prefix := strings.SplitN(expression, ".", 2)[0] + if !validPrefixes.Has(prefix) { + errs = errs.Also(apis.ErrInvalidValue( + fmt.Sprintf("invalid variable reference %q, must start with a valid prefix: params, tasks, finally, context, workspaces, or results; if you meant a shell variable, use ${VAR} instead", "$("+expression+")"), + "value", + ).ViaFieldKey("matrix.params", param.Name).ViaFieldIndex(fieldPath, idx)) + } + } + } + } + for i, include := range task.Matrix.Include { + for _, param := range include.Params { + if expressions, ok := param.GetVarSubstitutionExpressions(); ok { + for _, expression := range expressions { + prefix := strings.SplitN(expression, ".", 2)[0] + if !validPrefixes.Has(prefix) { + errs = errs.Also(apis.ErrInvalidValue( + fmt.Sprintf("invalid variable reference %q, must start with a valid prefix: params, tasks, finally, context, workspaces, or results; if you meant a shell variable, use ${VAR} instead", "$("+expression+")"), + "value", + ).ViaFieldKey("params", param.Name).ViaFieldIndex("matrix.include", i).ViaFieldIndex(fieldPath, idx)) + } + } + } + } + } + } + } + return errs +} + // findAndValidateResultRefsForMatrix checks that any result references to Matrixed PipelineTasks if consumed // by another PipelineTask that the entire array of results produced by a matrix is consumed in aggregate // since consuming a singular result produced by a matrix is currently not supported diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipelinerun_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipelinerun_types.go index 7cbd9cafc..3ab8f9edf 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipelinerun_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/pipelinerun_types.go @@ -37,6 +37,7 @@ import ( ) // +genclient +// +kubebuilder:object:root=true // +genreconciler:krshapedlogic=false // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -366,6 +367,9 @@ const ( // PipelineRunReasonStopping indicates that no new Tasks will be scheduled by the controller, and the // pipeline will stop once all running tasks complete their work PipelineRunReasonStopping PipelineRunReason = "PipelineRunStopping" + // PipelineRunReasonTimedOutRunningFinally indicates that the tasks timeout has been exceeded + // and no new DAG tasks will be scheduled, but final tasks are now running + PipelineRunReasonTimedOutRunningFinally PipelineRunReason = "PipelineRunTimeoutRunningFinally" // PipelineRunReasonCancelledRunningFinally indicates that pipeline has been gracefully cancelled // and no new Tasks will be scheduled by the controller, but final tasks are now running PipelineRunReasonCancelledRunningFinally PipelineRunReason = "CancelledRunningFinally" diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/swagger.json b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/swagger.json index bf1ad9866..a0964b1d4 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/swagger.json +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/swagger.json @@ -983,11 +983,11 @@ } }, "pipelineRef": { - "description": "PipelineRef is a reference to a pipeline definition Note: PipelineRef is in preview mode and not yet supported", + "description": "PipelineRef is a reference to a pipeline definition. This is an alpha field. You must set the \"enable-api-fields\" feature flag to \"alpha\" for this field to be supported. When enabled, the referenced Pipeline is executed as a child PipelineRun owned by the parent PipelineRun.", "$ref": "#/definitions/v1.PipelineRef" }, "pipelineSpec": { - "description": "PipelineSpec is a specification of a pipeline Note: PipelineSpec is in preview mode and not yet supported Specifying PipelineSpec can be disabled by setting `disable-inline-spec` feature flag. See Pipeline.spec (API version: tekton.dev/v1)", + "description": "PipelineSpec is a specification of a pipeline. This is an alpha field. You must set the \"enable-api-fields\" feature flag to \"alpha\" for this field to be supported. When enabled, the embedded Pipeline is executed as a child PipelineRun owned by the parent PipelineRun. Specifying PipelineSpec can be disabled by setting `disable-inline-spec` feature flag. See Pipeline.spec (API version: tekton.dev/v1)", "$ref": "#/definitions/v1.PipelineSpec" }, "retries": { diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_types.go index 05fb47f15..36dac6157 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_types.go @@ -26,6 +26,7 @@ import ( ) // +genclient +// +kubebuilder:object:root=true // +genclient:noStatus // +genreconciler:krshapedlogic=false // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_validation.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_validation.go index 5106a5faf..4dfc89965 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_validation.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/task_validation.go @@ -247,7 +247,21 @@ func ValidateStepResults(ctx context.Context, results []StepResult) (errs *apis. } // ValidateStepResultsVariables validates if the StepResults referenced in step script are defined in step's results. +// It only validates $(step.results..path) references. Task-level $(results..path) references +// are validated separately by validateTaskResultsVariables which has the task results context. func ValidateStepResultsVariables(ctx context.Context, results []StepResult, script string) (errs *apis.FieldError) { + resultsNames := sets.NewString() + for _, r := range results { + resultsNames.Insert(r.Name) + } + errs = errs.Also(substitution.ValidateNoReferencesToUnknownVariables(script, "step.results", resultsNames).ViaField("script")) + return errs +} + +// ValidateStepActionResultsVariables validates if the results referenced in a StepAction script are defined +// in the StepAction's results. Unlike ValidateStepResultsVariables, this also validates $(results..path) +// references since a standalone StepAction has no task-level result context. +func ValidateStepActionResultsVariables(ctx context.Context, results []StepResult, script string) (errs *apis.FieldError) { resultsNames := sets.NewString() for _, r := range results { resultsNames.Insert(r.Name) diff --git a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/taskrun_types.go b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/taskrun_types.go index f986f512f..d6f92c72f 100644 --- a/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/taskrun_types.go +++ b/vendor/github.com/tektoncd/pipeline/pkg/apis/pipeline/v1/taskrun_types.go @@ -440,6 +440,7 @@ type SidecarState struct { } // +genclient +// +kubebuilder:object:root=true // +genreconciler:krshapedlogic=false // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/vendor/github.com/vbatts/tar-split/LICENSE b/vendor/github.com/vbatts/tar-split/LICENSE deleted file mode 100644 index ca03685b1..000000000 --- a/vendor/github.com/vbatts/tar-split/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/common.go b/vendor/github.com/vbatts/tar-split/archive/tar/common.go deleted file mode 100644 index e687a08c9..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/common.go +++ /dev/null @@ -1,724 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package tar implements access to tar archives. -// -// Tape archives (tar) are a file format for storing a sequence of files that -// can be read and written in a streaming manner. -// This package aims to cover most variations of the format, -// including those produced by GNU and BSD tar tools. -package tar - -import ( - "errors" - "fmt" - "math" - "os" - "path" - "reflect" - "strconv" - "strings" - "time" -) - -// BUG: Use of the Uid and Gid fields in Header could overflow on 32-bit -// architectures. If a large value is encountered when decoding, the result -// stored in Header will be the truncated version. - -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errMissData = errors.New("archive/tar: sparse file references non-existent data") - errUnrefData = errors.New("archive/tar: sparse file contains unreferenced data") - errWriteHole = errors.New("archive/tar: write non-NUL byte in sparse hole") - errSparseTooLong = errors.New("archive/tar: sparse map too long") -) - -type headerError []string - -func (he headerError) Error() string { - const prefix = "archive/tar: cannot encode header" - var ss []string - for _, s := range he { - if s != "" { - ss = append(ss, s) - } - } - if len(ss) == 0 { - return prefix - } - return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and ")) -} - -// Type flags for Header.Typeflag. -const ( - // Type '0' indicates a regular file. - TypeReg = '0' - TypeRegA = '\x00' // Deprecated: Use TypeReg instead. - - // Type '1' to '6' are header-only flags and may not have a data body. - TypeLink = '1' // Hard link - TypeSymlink = '2' // Symbolic link - TypeChar = '3' // Character device node - TypeBlock = '4' // Block device node - TypeDir = '5' // Directory - TypeFifo = '6' // FIFO node - - // Type '7' is reserved. - TypeCont = '7' - - // Type 'x' is used by the PAX format to store key-value records that - // are only relevant to the next file. - // This package transparently handles these types. - TypeXHeader = 'x' - - // Type 'g' is used by the PAX format to store key-value records that - // are relevant to all subsequent files. - // This package only supports parsing and composing such headers, - // but does not currently support persisting the global state across files. - TypeXGlobalHeader = 'g' - - // Type 'S' indicates a sparse file in the GNU format. - TypeGNUSparse = 'S' - - // Types 'L' and 'K' are used by the GNU format for a meta file - // used to store the path or link name for the next file. - // This package transparently handles these types. - TypeGNULongName = 'L' - TypeGNULongLink = 'K' -) - -// Keywords for PAX extended header records. -const ( - paxNone = "" // Indicates that no PAX key is suitable - paxPath = "path" - paxLinkpath = "linkpath" - paxSize = "size" - paxUid = "uid" - paxGid = "gid" - paxUname = "uname" - paxGname = "gname" - paxMtime = "mtime" - paxAtime = "atime" - paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid - paxCharset = "charset" // Currently unused - paxComment = "comment" // Currently unused - - paxSchilyXattr = "SCHILY.xattr." - - // Keywords for GNU sparse files in a PAX extended header. - paxGNUSparse = "GNU.sparse." - paxGNUSparseNumBlocks = "GNU.sparse.numblocks" - paxGNUSparseOffset = "GNU.sparse.offset" - paxGNUSparseNumBytes = "GNU.sparse.numbytes" - paxGNUSparseMap = "GNU.sparse.map" - paxGNUSparseName = "GNU.sparse.name" - paxGNUSparseMajor = "GNU.sparse.major" - paxGNUSparseMinor = "GNU.sparse.minor" - paxGNUSparseSize = "GNU.sparse.size" - paxGNUSparseRealSize = "GNU.sparse.realsize" -) - -// basicKeys is a set of the PAX keys for which we have built-in support. -// This does not contain "charset" or "comment", which are both PAX-specific, -// so adding them as first-class features of Header is unlikely. -// Users can use the PAXRecords field to set it themselves. -var basicKeys = map[string]bool{ - paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true, - paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true, -} - -// A Header represents a single header in a tar archive. -// Some fields may not be populated. -// -// For forward compatibility, users that retrieve a Header from Reader.Next, -// mutate it in some ways, and then pass it back to Writer.WriteHeader -// should do so by creating a new Header and copying the fields -// that they are interested in preserving. -type Header struct { - // Typeflag is the type of header entry. - // The zero value is automatically promoted to either TypeReg or TypeDir - // depending on the presence of a trailing slash in Name. - Typeflag byte - - Name string // Name of file entry - Linkname string // Target name of link (valid for TypeLink or TypeSymlink) - - Size int64 // Logical file size in bytes - Mode int64 // Permission and mode bits - Uid int // User ID of owner - Gid int // Group ID of owner - Uname string // User name of owner - Gname string // Group name of owner - - // If the Format is unspecified, then Writer.WriteHeader rounds ModTime - // to the nearest second and ignores the AccessTime and ChangeTime fields. - // - // To use AccessTime or ChangeTime, specify the Format as PAX or GNU. - // To use sub-second resolution, specify the Format as PAX. - ModTime time.Time // Modification time - AccessTime time.Time // Access time (requires either PAX or GNU support) - ChangeTime time.Time // Change time (requires either PAX or GNU support) - - Devmajor int64 // Major device number (valid for TypeChar or TypeBlock) - Devminor int64 // Minor device number (valid for TypeChar or TypeBlock) - - // Xattrs stores extended attributes as PAX records under the - // "SCHILY.xattr." namespace. - // - // The following are semantically equivalent: - // h.Xattrs[key] = value - // h.PAXRecords["SCHILY.xattr."+key] = value - // - // When Writer.WriteHeader is called, the contents of Xattrs will take - // precedence over those in PAXRecords. - // - // Deprecated: Use PAXRecords instead. - Xattrs map[string]string - - // PAXRecords is a map of PAX extended header records. - // - // User-defined records should have keys of the following form: - // VENDOR.keyword - // Where VENDOR is some namespace in all uppercase, and keyword may - // not contain the '=' character (e.g., "GOLANG.pkg.version"). - // The key and value should be non-empty UTF-8 strings. - // - // When Writer.WriteHeader is called, PAX records derived from the - // other fields in Header take precedence over PAXRecords. - PAXRecords map[string]string - - // Format specifies the format of the tar header. - // - // This is set by Reader.Next as a best-effort guess at the format. - // Since the Reader liberally reads some non-compliant files, - // it is possible for this to be FormatUnknown. - // - // If the format is unspecified when Writer.WriteHeader is called, - // then it uses the first format (in the order of USTAR, PAX, GNU) - // capable of encoding this Header (see Format). - Format Format -} - -// sparseEntry represents a Length-sized fragment at Offset in the file. -type sparseEntry struct{ Offset, Length int64 } - -func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length } - -// A sparse file can be represented as either a sparseDatas or a sparseHoles. -// As long as the total size is known, they are equivalent and one can be -// converted to the other form and back. The various tar formats with sparse -// file support represent sparse files in the sparseDatas form. That is, they -// specify the fragments in the file that has data, and treat everything else as -// having zero bytes. As such, the encoding and decoding logic in this package -// deals with sparseDatas. -// -// However, the external API uses sparseHoles instead of sparseDatas because the -// zero value of sparseHoles logically represents a normal file (i.e., there are -// no holes in it). On the other hand, the zero value of sparseDatas implies -// that the file has no data in it, which is rather odd. -// -// As an example, if the underlying raw file contains the 10-byte data: -// var compactFile = "abcdefgh" -// -// And the sparse map has the following entries: -// var spd sparseDatas = []sparseEntry{ -// {Offset: 2, Length: 5}, // Data fragment for 2..6 -// {Offset: 18, Length: 3}, // Data fragment for 18..20 -// } -// var sph sparseHoles = []sparseEntry{ -// {Offset: 0, Length: 2}, // Hole fragment for 0..1 -// {Offset: 7, Length: 11}, // Hole fragment for 7..17 -// {Offset: 21, Length: 4}, // Hole fragment for 21..24 -// } -// -// Then the content of the resulting sparse file with a Header.Size of 25 is: -// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 -type ( - sparseDatas []sparseEntry - sparseHoles []sparseEntry -) - -// validateSparseEntries reports whether sp is a valid sparse map. -// It does not matter whether sp represents data fragments or hole fragments. -func validateSparseEntries(sp []sparseEntry, size int64) bool { - // Validate all sparse entries. These are the same checks as performed by - // the BSD tar utility. - if size < 0 { - return false - } - var pre sparseEntry - for _, cur := range sp { - switch { - case cur.Offset < 0 || cur.Length < 0: - return false // Negative values are never okay - case cur.Offset > math.MaxInt64-cur.Length: - return false // Integer overflow with large length - case cur.endOffset() > size: - return false // Region extends beyond the actual size - case pre.endOffset() > cur.Offset: - return false // Regions cannot overlap and must be in order - } - pre = cur - } - return true -} - -// alignSparseEntries mutates src and returns dst where each fragment's -// starting offset is aligned up to the nearest block edge, and each -// ending offset is aligned down to the nearest block edge. -// -// Even though the Go tar Reader and the BSD tar utility can handle entries -// with arbitrary offsets and lengths, the GNU tar utility can only handle -// offsets and lengths that are multiples of blockSize. -func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - for _, s := range src { - pos, end := s.Offset, s.endOffset() - pos += blockPadding(+pos) // Round-up to nearest blockSize - if end != size { - end -= blockPadding(-end) // Round-down to nearest blockSize - } - if pos < end { - dst = append(dst, sparseEntry{Offset: pos, Length: end - pos}) - } - } - return dst -} - -// invertSparseEntries converts a sparse map from one form to the other. -// If the input is sparseHoles, then it will output sparseDatas and vice-versa. -// The input must have been already validated. -// -// This function mutates src and returns a normalized map where: -// * adjacent fragments are coalesced together -// * only the last fragment may be empty -// * the endOffset of the last fragment is the total size -func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - var pre sparseEntry - for _, cur := range src { - if cur.Length == 0 { - continue // Skip empty fragments - } - pre.Length = cur.Offset - pre.Offset - if pre.Length > 0 { - dst = append(dst, pre) // Only add non-empty fragments - } - pre.Offset = cur.endOffset() - } - pre.Length = size - pre.Offset // Possibly the only empty fragment - return append(dst, pre) -} - -// fileState tracks the number of logical (includes sparse holes) and physical -// (actual in tar archive) bytes remaining for the current file. -// -// Invariant: LogicalRemaining >= PhysicalRemaining -type fileState interface { - LogicalRemaining() int64 - PhysicalRemaining() int64 -} - -// allowedFormats determines which formats can be used. -// The value returned is the logical OR of multiple possible formats. -// If the value is FormatUnknown, then the input Header cannot be encoded -// and an error is returned explaining why. -// -// As a by-product of checking the fields, this function returns paxHdrs, which -// contain all fields that could not be directly encoded. -// A value receiver ensures that this method does not mutate the source Header. -func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) { - format = FormatUSTAR | FormatPAX | FormatGNU - paxHdrs = make(map[string]string) - - var whyNoUSTAR, whyNoPAX, whyNoGNU string - var preferPAX bool // Prefer PAX over USTAR - verifyString := func(s string, size int, name, paxKey string) { - // NUL-terminator is optional for path and linkpath. - // Technically, it is required for uname and gname, - // but neither GNU nor BSD tar checks for it. - tooLong := len(s) > size - allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath - if hasNUL(s) || (tooLong && !allowLongGNU) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s) - format.mustNotBe(FormatGNU) - } - if !isASCII(s) || tooLong { - canSplitUSTAR := paxKey == paxPath - if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s) - format.mustNotBe(FormatUSTAR) - } - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = s - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == s { - paxHdrs[paxKey] = v - } - } - verifyNumeric := func(n int64, size int, name, paxKey string) { - if !fitsInBase256(size, n) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n) - format.mustNotBe(FormatGNU) - } - if !fitsInOctal(size, n) { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n) - format.mustNotBe(FormatUSTAR) - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = strconv.FormatInt(n, 10) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) { - paxHdrs[paxKey] = v - } - } - verifyTime := func(ts time.Time, size int, name, paxKey string) { - if ts.IsZero() { - return // Always okay - } - if !fitsInBase256(size, ts.Unix()) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts) - format.mustNotBe(FormatGNU) - } - isMtime := paxKey == paxMtime - fitsOctal := fitsInOctal(size, ts.Unix()) - if (isMtime && !fitsOctal) || !isMtime { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts) - format.mustNotBe(FormatUSTAR) - } - needsNano := ts.Nanosecond() != 0 - if !isMtime || !fitsOctal || needsNano { - preferPAX = true // USTAR may truncate sub-second measurements - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = formatPAXTime(ts) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) { - paxHdrs[paxKey] = v - } - } - - // Check basic fields. - var blk block - v7 := blk.V7() - ustar := blk.USTAR() - gnu := blk.GNU() - verifyString(h.Name, len(v7.Name()), "Name", paxPath) - verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath) - verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname) - verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname) - verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone) - verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid) - verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid) - verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize) - verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone) - verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone) - verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime) - verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime) - verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime) - - // Check for header-only types. - var whyOnlyPAX, whyOnlyGNU string - switch h.Typeflag { - case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse: - // Exclude TypeLink and TypeSymlink, since they may reference directories. - if strings.HasSuffix(h.Name, "/") { - return FormatUnknown, nil, headerError{"filename may not have trailing slash"} - } - case TypeXHeader, TypeGNULongName, TypeGNULongLink: - return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"} - case TypeXGlobalHeader: - h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format} - if !reflect.DeepEqual(h, h2) { - return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"} - } - whyOnlyPAX = "only PAX supports TypeXGlobalHeader" - format.mayOnlyBe(FormatPAX) - } - if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 { - return FormatUnknown, nil, headerError{"negative size on header-only type"} - } - - // Check PAX records. - if len(h.Xattrs) > 0 { - for k, v := range h.Xattrs { - paxHdrs[paxSchilyXattr+k] = v - } - whyOnlyPAX = "only PAX supports Xattrs" - format.mayOnlyBe(FormatPAX) - } - if len(h.PAXRecords) > 0 { - for k, v := range h.PAXRecords { - switch _, exists := paxHdrs[k]; { - case exists: - continue // Do not overwrite existing records - case h.Typeflag == TypeXGlobalHeader: - paxHdrs[k] = v // Copy all records - case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse): - paxHdrs[k] = v // Ignore local records that may conflict - } - } - whyOnlyPAX = "only PAX supports PAXRecords" - format.mayOnlyBe(FormatPAX) - } - for k, v := range paxHdrs { - if !validPAXRecord(k, v) { - return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)} - } - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Check sparse files. - if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse { - if isHeaderOnlyType(h.Typeflag) { - return FormatUnknown, nil, headerError{"header-only type cannot be sparse"} - } - if !validateSparseEntries(h.SparseHoles, h.Size) { - return FormatUnknown, nil, headerError{"invalid sparse holes"} - } - if h.Typeflag == TypeGNUSparse { - whyOnlyGNU = "only GNU supports TypeGNUSparse" - format.mayOnlyBe(FormatGNU) - } else { - whyNoGNU = "GNU supports sparse files only with TypeGNUSparse" - format.mustNotBe(FormatGNU) - } - whyNoUSTAR = "USTAR does not support sparse files" - format.mustNotBe(FormatUSTAR) - } - */ - - // Check desired format. - if wantFormat := h.Format; wantFormat != FormatUnknown { - if wantFormat.has(FormatPAX) && !preferPAX { - wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too - } - format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted - } - if format == FormatUnknown { - switch h.Format { - case FormatUSTAR: - err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU} - case FormatPAX: - err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU} - case FormatGNU: - err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX} - default: - err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU} - } - } - return format, paxHdrs, err -} - -// FileInfo returns an os.FileInfo for the Header. -func (h *Header) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - h *Header -} - -func (fi headerFileInfo) Size() int64 { return fi.h.Size } -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } -func (fi headerFileInfo) Sys() interface{} { return fi.h } - -// Name returns the base name of the file. -func (fi headerFileInfo) Name() string { - if fi.IsDir() { - return path.Base(path.Clean(fi.h.Name)) - } - return path.Base(fi.h.Name) -} - -// Mode returns the permission and mode bits for the headerFileInfo. -func (fi headerFileInfo) Mode() (mode os.FileMode) { - // Set file permission bits. - mode = os.FileMode(fi.h.Mode).Perm() - - // Set setuid, setgid and sticky bits. - if fi.h.Mode&c_ISUID != 0 { - mode |= os.ModeSetuid - } - if fi.h.Mode&c_ISGID != 0 { - mode |= os.ModeSetgid - } - if fi.h.Mode&c_ISVTX != 0 { - mode |= os.ModeSticky - } - - // Set file mode bits; clear perm, setuid, setgid, and sticky bits. - switch m := os.FileMode(fi.h.Mode) &^ 07777; m { - case c_ISDIR: - mode |= os.ModeDir - case c_ISFIFO: - mode |= os.ModeNamedPipe - case c_ISLNK: - mode |= os.ModeSymlink - case c_ISBLK: - mode |= os.ModeDevice - case c_ISCHR: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case c_ISSOCK: - mode |= os.ModeSocket - } - - switch fi.h.Typeflag { - case TypeSymlink: - mode |= os.ModeSymlink - case TypeChar: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case TypeBlock: - mode |= os.ModeDevice - case TypeDir: - mode |= os.ModeDir - case TypeFifo: - mode |= os.ModeNamedPipe - } - - return mode -} - -// sysStat, if non-nil, populates h from system-dependent fields of fi. -var sysStat func(fi os.FileInfo, h *Header) error - -const ( - // Mode constants from the USTAR spec: - // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) - - // Common Unix mode constants; these are not defined in any common tar standard. - // Header.FileInfo understands these, but FileInfoHeader will never produce these. - c_ISDIR = 040000 // Directory - c_ISFIFO = 010000 // FIFO - c_ISREG = 0100000 // Regular file - c_ISLNK = 0120000 // Symbolic link - c_ISBLK = 060000 // Block special file - c_ISCHR = 020000 // Character special file - c_ISSOCK = 0140000 // Socket -) - -// FileInfoHeader creates a partially-populated Header from fi. -// If fi describes a symlink, FileInfoHeader records link as the link target. -// If fi describes a directory, a slash is appended to the name. -// -// Since os.FileInfo's Name method only returns the base name of -// the file it describes, it may be necessary to modify Header.Name -// to provide the full path name of the file. -func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { - if fi == nil { - return nil, errors.New("archive/tar: FileInfo is nil") - } - fm := fi.Mode() - h := &Header{ - Name: fi.Name(), - ModTime: fi.ModTime(), - Mode: int64(fm.Perm()), // or'd with c_IS* constants later - } - switch { - case fm.IsRegular(): - h.Typeflag = TypeReg - h.Size = fi.Size() - case fi.IsDir(): - h.Typeflag = TypeDir - h.Name += "/" - case fm&os.ModeSymlink != 0: - h.Typeflag = TypeSymlink - h.Linkname = link - case fm&os.ModeDevice != 0: - if fm&os.ModeCharDevice != 0 { - h.Typeflag = TypeChar - } else { - h.Typeflag = TypeBlock - } - case fm&os.ModeNamedPipe != 0: - h.Typeflag = TypeFifo - case fm&os.ModeSocket != 0: - return nil, fmt.Errorf("archive/tar: sockets not supported") - default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) - } - if fm&os.ModeSetuid != 0 { - h.Mode |= c_ISUID - } - if fm&os.ModeSetgid != 0 { - h.Mode |= c_ISGID - } - if fm&os.ModeSticky != 0 { - h.Mode |= c_ISVTX - } - // If possible, populate additional fields from OS-specific - // FileInfo fields. - if sys, ok := fi.Sys().(*Header); ok { - // This FileInfo came from a Header (not the OS). Use the - // original Header to populate all remaining fields. - h.Uid = sys.Uid - h.Gid = sys.Gid - h.Uname = sys.Uname - h.Gname = sys.Gname - h.AccessTime = sys.AccessTime - h.ChangeTime = sys.ChangeTime - if sys.Xattrs != nil { - h.Xattrs = make(map[string]string) - for k, v := range sys.Xattrs { - h.Xattrs[k] = v - } - } - if sys.Typeflag == TypeLink { - // hard link - h.Typeflag = TypeLink - h.Size = 0 - h.Linkname = sys.Linkname - } - if sys.PAXRecords != nil { - h.PAXRecords = make(map[string]string) - for k, v := range sys.PAXRecords { - h.PAXRecords[k] = v - } - } - } - if sysStat != nil { - return h, sysStat(fi, h) - } - return h, nil -} - -// isHeaderOnlyType checks if the given type flag is of the type that has no -// data section even if a size is specified. -func isHeaderOnlyType(flag byte) bool { - switch flag { - case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo: - return true - default: - return false - } -} - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/format.go b/vendor/github.com/vbatts/tar-split/archive/tar/format.go deleted file mode 100644 index 60977980c..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/format.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import "strings" - -// Format represents the tar archive format. -// -// The original tar format was introduced in Unix V7. -// Since then, there have been multiple competing formats attempting to -// standardize or extend the V7 format to overcome its limitations. -// The most common formats are the USTAR, PAX, and GNU formats, -// each with their own advantages and limitations. -// -// The following table captures the capabilities of each format: -// -// | USTAR | PAX | GNU -// ------------------+--------+-----------+---------- -// Name | 256B | unlimited | unlimited -// Linkname | 100B | unlimited | unlimited -// Size | uint33 | unlimited | uint89 -// Mode | uint21 | uint21 | uint57 -// Uid/Gid | uint21 | unlimited | uint57 -// Uname/Gname | 32B | unlimited | 32B -// ModTime | uint33 | unlimited | int89 -// AccessTime | n/a | unlimited | int89 -// ChangeTime | n/a | unlimited | int89 -// Devmajor/Devminor | uint21 | uint21 | uint57 -// ------------------+--------+-----------+---------- -// string encoding | ASCII | UTF-8 | binary -// sub-second times | no | yes | no -// sparse files | no | yes | yes -// -// The table's upper portion shows the Header fields, where each format reports -// the maximum number of bytes allowed for each string field and -// the integer type used to store each numeric field -// (where timestamps are stored as the number of seconds since the Unix epoch). -// -// The table's lower portion shows specialized features of each format, -// such as supported string encodings, support for sub-second timestamps, -// or support for sparse files. -// -// The Writer currently provides no support for sparse files. -type Format int - -// Constants to identify various tar formats. -const ( - // Deliberately hide the meaning of constants from public API. - _ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc... - - // FormatUnknown indicates that the format is unknown. - FormatUnknown - - // The format of the original Unix V7 tar tool prior to standardization. - formatV7 - - // FormatUSTAR represents the USTAR header format defined in POSIX.1-1988. - // - // While this format is compatible with most tar readers, - // the format has several limitations making it unsuitable for some usages. - // Most notably, it cannot support sparse files, files larger than 8GiB, - // filenames larger than 256 characters, and non-ASCII filenames. - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - FormatUSTAR - - // FormatPAX represents the PAX header format defined in POSIX.1-2001. - // - // PAX extends USTAR by writing a special file with Typeflag TypeXHeader - // preceding the original header. This file contains a set of key-value - // records, which are used to overcome USTAR's shortcomings, in addition to - // providing the ability to have sub-second resolution for timestamps. - // - // Some newer formats add their own extensions to PAX by defining their - // own keys and assigning certain semantic meaning to the associated values. - // For example, sparse file support in PAX is implemented using keys - // defined by the GNU manual (e.g., "GNU.sparse.map"). - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html - FormatPAX - - // FormatGNU represents the GNU header format. - // - // The GNU header format is older than the USTAR and PAX standards and - // is not compatible with them. The GNU format supports - // arbitrary file sizes, filenames of arbitrary encoding and length, - // sparse files, and other features. - // - // It is recommended that PAX be chosen over GNU unless the target - // application can only parse GNU formatted archives. - // - // Reference: - // https://www.gnu.org/software/tar/manual/html_node/Standard.html - FormatGNU - - // Schily's tar format, which is incompatible with USTAR. - // This does not cover STAR extensions to the PAX format; these fall under - // the PAX format. - formatSTAR - - formatMax -) - -func (f Format) has(f2 Format) bool { return f&f2 != 0 } -func (f *Format) mayBe(f2 Format) { *f |= f2 } -func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 } -func (f *Format) mustNotBe(f2 Format) { *f &^= f2 } - -var formatNames = map[Format]string{ - formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR", -} - -func (f Format) String() string { - var ss []string - for f2 := Format(1); f2 < formatMax; f2 <<= 1 { - if f.has(f2) { - ss = append(ss, formatNames[f2]) - } - } - switch len(ss) { - case 0: - return "" - case 1: - return ss[0] - default: - return "(" + strings.Join(ss, " | ") + ")" - } -} - -// Magics used to identify various formats. -const ( - magicGNU, versionGNU = "ustar ", " \x00" - magicUSTAR, versionUSTAR = "ustar\x00", "00" - trailerSTAR = "tar\x00" -) - -// Size constants from various tar specifications. -const ( - blockSize = 512 // Size of each block in a tar stream - nameSize = 100 // Max length of the name field in USTAR format - prefixSize = 155 // Max length of the prefix field in USTAR format - - // Max length of a special file (PAX header, GNU long name or link). - // This matches the limit used by libarchive. - maxSpecialFileSize = 1 << 20 -) - -// blockPadding computes the number of bytes needed to pad offset up to the -// nearest block edge where 0 <= n < blockSize. -func blockPadding(offset int64) (n int64) { - return -offset & (blockSize - 1) -} - -var zeroBlock block - -type block [blockSize]byte - -// Convert block to any number of formats. -func (b *block) V7() *headerV7 { return (*headerV7)(b) } -func (b *block) GNU() *headerGNU { return (*headerGNU)(b) } -func (b *block) STAR() *headerSTAR { return (*headerSTAR)(b) } -func (b *block) USTAR() *headerUSTAR { return (*headerUSTAR)(b) } -func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) } - -// GetFormat checks that the block is a valid tar header based on the checksum. -// It then attempts to guess the specific format based on magic values. -// If the checksum fails, then FormatUnknown is returned. -func (b *block) GetFormat() Format { - // Verify checksum. - var p parser - value := p.parseOctal(b.V7().Chksum()) - chksum1, chksum2 := b.ComputeChecksum() - if p.err != nil || (value != chksum1 && value != chksum2) { - return FormatUnknown - } - - // Guess the magic values. - magic := string(b.USTAR().Magic()) - version := string(b.USTAR().Version()) - trailer := string(b.STAR().Trailer()) - switch { - case magic == magicUSTAR && trailer == trailerSTAR: - return formatSTAR - case magic == magicUSTAR: - return FormatUSTAR | FormatPAX - case magic == magicGNU && version == versionGNU: - return FormatGNU - default: - return formatV7 - } -} - -// SetFormat writes the magic values necessary for specified format -// and then updates the checksum accordingly. -func (b *block) SetFormat(format Format) { - // Set the magic values. - switch { - case format.has(formatV7): - // Do nothing. - case format.has(FormatGNU): - copy(b.GNU().Magic(), magicGNU) - copy(b.GNU().Version(), versionGNU) - case format.has(formatSTAR): - copy(b.STAR().Magic(), magicUSTAR) - copy(b.STAR().Version(), versionUSTAR) - copy(b.STAR().Trailer(), trailerSTAR) - case format.has(FormatUSTAR | FormatPAX): - copy(b.USTAR().Magic(), magicUSTAR) - copy(b.USTAR().Version(), versionUSTAR) - default: - panic("invalid format") - } - - // Update checksum. - // This field is special in that it is terminated by a NULL then space. - var f formatter - field := b.V7().Chksum() - chksum, _ := b.ComputeChecksum() // Possible values are 256..128776 - f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143 - field[7] = ' ' -} - -// ComputeChecksum computes the checksum for the header block. -// POSIX specifies a sum of the unsigned byte values, but the Sun tar used -// signed byte values. -// We compute and return both. -func (b *block) ComputeChecksum() (unsigned, signed int64) { - for i, c := range b { - if 148 <= i && i < 156 { - c = ' ' // Treat the checksum field itself as all spaces. - } - unsigned += int64(c) - signed += int64(int8(c)) - } - return unsigned, signed -} - -// Reset clears the block with all zeros. -func (b *block) Reset() { - *b = block{} -} - -type headerV7 [blockSize]byte - -func (h *headerV7) Name() []byte { return h[000:][:100] } -func (h *headerV7) Mode() []byte { return h[100:][:8] } -func (h *headerV7) UID() []byte { return h[108:][:8] } -func (h *headerV7) GID() []byte { return h[116:][:8] } -func (h *headerV7) Size() []byte { return h[124:][:12] } -func (h *headerV7) ModTime() []byte { return h[136:][:12] } -func (h *headerV7) Chksum() []byte { return h[148:][:8] } -func (h *headerV7) TypeFlag() []byte { return h[156:][:1] } -func (h *headerV7) LinkName() []byte { return h[157:][:100] } - -type headerGNU [blockSize]byte - -func (h *headerGNU) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerGNU) Magic() []byte { return h[257:][:6] } -func (h *headerGNU) Version() []byte { return h[263:][:2] } -func (h *headerGNU) UserName() []byte { return h[265:][:32] } -func (h *headerGNU) GroupName() []byte { return h[297:][:32] } -func (h *headerGNU) DevMajor() []byte { return h[329:][:8] } -func (h *headerGNU) DevMinor() []byte { return h[337:][:8] } -func (h *headerGNU) AccessTime() []byte { return h[345:][:12] } -func (h *headerGNU) ChangeTime() []byte { return h[357:][:12] } -func (h *headerGNU) Sparse() sparseArray { return (sparseArray)(h[386:][:24*4+1]) } -func (h *headerGNU) RealSize() []byte { return h[483:][:12] } - -type headerSTAR [blockSize]byte - -func (h *headerSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerSTAR) Version() []byte { return h[263:][:2] } -func (h *headerSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerSTAR) Prefix() []byte { return h[345:][:131] } -func (h *headerSTAR) AccessTime() []byte { return h[476:][:12] } -func (h *headerSTAR) ChangeTime() []byte { return h[488:][:12] } -func (h *headerSTAR) Trailer() []byte { return h[508:][:4] } - -type headerUSTAR [blockSize]byte - -func (h *headerUSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerUSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerUSTAR) Version() []byte { return h[263:][:2] } -func (h *headerUSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerUSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerUSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerUSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] } - -type sparseArray []byte - -func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) } -func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] } -func (s sparseArray) MaxEntries() int { return len(s) / 24 } - -type sparseElem []byte - -func (s sparseElem) Offset() []byte { return s[00:][:12] } -func (s sparseElem) Length() []byte { return s[12:][:12] } diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go b/vendor/github.com/vbatts/tar-split/archive/tar/reader.go deleted file mode 100644 index a645c4160..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go +++ /dev/null @@ -1,944 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "io" - "strconv" - "strings" - "time" -) - -// Reader provides sequential access to the contents of a tar archive. -// Reader.Next advances to the next file in the archive (including the first), -// and then Reader can be treated as an io.Reader to access the file's data. -type Reader struct { - r io.Reader - pad int64 // Amount of padding (ignored) after current file entry - curr fileReader // Reader for current file entry - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Reader to - // ensure that this error is sticky. - err error - - RawAccounting bool // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this. - rawBytes *bytes.Buffer // last raw bits -} - -type fileReader interface { - io.Reader - fileState - - WriteTo(io.Writer) (int64, error) -} - -// RawBytes accesses the raw bytes of the archive, apart from the file payload itself. -// This includes the header and padding. -// -// # This call resets the current rawbytes buffer -// -// Only when RawAccounting is enabled, otherwise this returns nil -func (tr *Reader) RawBytes() []byte { - if !tr.RawAccounting { - return nil - } - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } - defer tr.rawBytes.Reset() // if we've read them, then flush them. - - return tr.rawBytes.Bytes() - -} - -// ExpectedPadding returns the number of bytes of padding expected after the last header returned by Next() -func (tr *Reader) ExpectedPadding() int64 { - return tr.pad -} - -// NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { - return &Reader{r: r, curr: ®FileReader{r, 0}} -} - -// Next advances to the next entry in the tar archive. -// The Header.Size determines how many bytes can be read for the next file. -// Any remaining data in the current file is automatically discarded. -// -// io.EOF is returned at the end of the input. -func (tr *Reader) Next() (*Header, error) { - if tr.err != nil { - return nil, tr.err - } - hdr, err := tr.next() - tr.err = err - return hdr, err -} - -func (tr *Reader) next() (*Header, error) { - var paxHdrs map[string]string - var gnuLongName, gnuLongLink string - - if tr.RawAccounting { - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } else { - tr.rawBytes.Reset() - } - } - - // Externally, Next iterates through the tar archive as if it is a series of - // files. Internally, the tar format often uses fake "files" to add meta - // data that describes the next file. These meta data "files" should not - // normally be visible to the outside. As such, this loop iterates through - // one or more "header files" until it finds a "normal file". - format := FormatUSTAR | FormatPAX | FormatGNU - for { - // Discard the remainder of the file and any padding. - if err := discard(tr, tr.curr.PhysicalRemaining()); err != nil { - return nil, err - } - n, err := tryReadFull(tr.r, tr.blk[:tr.pad]) - if err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(tr.blk[:n]) - } - tr.pad = 0 - - hdr, rawHdr, err := tr.readHeader() - if err != nil { - return nil, err - } - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - format.mayOnlyBe(hdr.Format) - - // Check for PAX/GNU special headers and files. - switch hdr.Typeflag { - case TypeXHeader, TypeXGlobalHeader: - format.mayOnlyBe(FormatPAX) - paxHdrs, err = parsePAX(tr) - if err != nil { - return nil, err - } - if hdr.Typeflag == TypeXGlobalHeader { - if err = mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - return &Header{ - Name: hdr.Name, - Typeflag: hdr.Typeflag, - Xattrs: hdr.Xattrs, - PAXRecords: hdr.PAXRecords, - Format: format, - }, nil - } - continue // This is a meta header affecting the next header - case TypeGNULongName, TypeGNULongLink: - format.mayOnlyBe(FormatGNU) - realname, err := readSpecialFile(tr) - if err != nil { - return nil, err - } - - if tr.RawAccounting { - tr.rawBytes.Write(realname) - } - - var p parser - switch hdr.Typeflag { - case TypeGNULongName: - gnuLongName = p.parseString(realname) - case TypeGNULongLink: - gnuLongLink = p.parseString(realname) - } - continue // This is a meta header affecting the next header - default: - // The old GNU sparse format is handled here since it is technically - // just a regular file with additional attributes. - - if err := mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - if gnuLongName != "" { - hdr.Name = gnuLongName - } - if gnuLongLink != "" { - hdr.Linkname = gnuLongLink - } - if hdr.Typeflag == TypeRegA { - if strings.HasSuffix(hdr.Name, "/") { - hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories - } else { - hdr.Typeflag = TypeReg - } - } - - // The extended headers may have updated the size. - // Thus, setup the regFileReader again after merging PAX headers. - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - - // Sparse formats rely on being able to read from the logical data - // section; there must be a preceding call to handleRegularFile. - if err := tr.handleSparseFile(hdr, rawHdr); err != nil { - return nil, err - } - - // Set the final guess at the format. - if format.has(FormatUSTAR) && format.has(FormatPAX) { - format.mayOnlyBe(FormatUSTAR) - } - hdr.Format = format - return hdr, nil // This is a file, so stop - } - } -} - -// handleRegularFile sets up the current file reader and padding such that it -// can only read the following logical data section. It will properly handle -// special headers that contain no data section. -func (tr *Reader) handleRegularFile(hdr *Header) error { - nb := hdr.Size - if isHeaderOnlyType(hdr.Typeflag) { - nb = 0 - } - if nb < 0 { - return ErrHeader - } - - tr.pad = blockPadding(nb) - tr.curr = ®FileReader{r: tr.r, nb: nb} - return nil -} - -// handleSparseFile checks if the current file is a sparse format of any type -// and sets the curr reader appropriately. -func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error { - var spd sparseDatas - var err error - if hdr.Typeflag == TypeGNUSparse { - spd, err = tr.readOldGNUSparseMap(hdr, rawHdr) - } else { - spd, err = tr.readGNUSparsePAXHeaders(hdr) - } - - // If sp is non-nil, then this is a sparse file. - // Note that it is possible for len(sp) == 0. - if err == nil && spd != nil { - if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) { - return ErrHeader - } - sph := invertSparseEntries(spd, hdr.Size) - tr.curr = &sparseFileReader{tr.curr, sph, 0} - } - return err -} - -// readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. -// If they are found, then this function reads the sparse map and returns it. -// This assumes that 0.0 headers have already been converted to 0.1 headers -// by the PAX header parsing logic. -func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) { - // Identify the version of GNU headers. - var is1x0 bool - major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor] - switch { - case major == "0" && (minor == "0" || minor == "1"): - is1x0 = false - case major == "1" && minor == "0": - is1x0 = true - case major != "" || minor != "": - return nil, nil // Unknown GNU sparse PAX version - case hdr.PAXRecords[paxGNUSparseMap] != "": - is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess - default: - return nil, nil // Not a PAX format GNU sparse file. - } - hdr.Format.mayOnlyBe(FormatPAX) - - // Update hdr from GNU sparse PAX headers. - if name := hdr.PAXRecords[paxGNUSparseName]; name != "" { - hdr.Name = name - } - size := hdr.PAXRecords[paxGNUSparseSize] - if size == "" { - size = hdr.PAXRecords[paxGNUSparseRealSize] - } - if size != "" { - n, err := strconv.ParseInt(size, 10, 64) - if err != nil { - return nil, ErrHeader - } - hdr.Size = n - } - - // Read the sparse map according to the appropriate format. - if is1x0 { - return readGNUSparseMap1x0(tr.curr) - } - return readGNUSparseMap0x1(hdr.PAXRecords) -} - -// mergePAX merges paxHdrs into hdr for all relevant fields of Header. -func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { - for k, v := range paxHdrs { - if v == "" { - continue // Keep the original USTAR value - } - var id64 int64 - switch k { - case paxPath: - hdr.Name = v - case paxLinkpath: - hdr.Linkname = v - case paxUname: - hdr.Uname = v - case paxGname: - hdr.Gname = v - case paxUid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Uid = int(id64) // Integer overflow possible - case paxGid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Gid = int(id64) // Integer overflow possible - case paxAtime: - hdr.AccessTime, err = parsePAXTime(v) - case paxMtime: - hdr.ModTime, err = parsePAXTime(v) - case paxCtime: - hdr.ChangeTime, err = parsePAXTime(v) - case paxSize: - hdr.Size, err = strconv.ParseInt(v, 10, 64) - default: - if strings.HasPrefix(k, paxSchilyXattr) { - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) - } - hdr.Xattrs[k[len(paxSchilyXattr):]] = v - } - } - if err != nil { - return ErrHeader - } - } - hdr.PAXRecords = paxHdrs - return nil -} - -// parsePAX parses PAX headers. -// If an extended header (type 'x') is invalid, ErrHeader is returned -func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := readSpecialFile(r) - if err != nil { - return nil, err - } - // leaving this function for io.Reader makes it more testable - if tr, ok := r.(*Reader); ok && tr.RawAccounting { - if _, err = tr.rawBytes.Write(buf); err != nil { - return nil, err - } - } - sbuf := string(buf) - - // For GNU PAX sparse format 0.0 support. - // This function transforms the sparse format 0.0 headers into format 0.1 - // headers since 0.0 headers were not PAX compliant. - var sparseMap []string - - paxHdrs := make(map[string]string) - for len(sbuf) > 0 { - key, value, residual, err := parsePAXRecord(sbuf) - if err != nil { - return nil, ErrHeader - } - sbuf = residual - - switch key { - case paxGNUSparseOffset, paxGNUSparseNumBytes: - // Validate sparse header order and value. - if (len(sparseMap)%2 == 0 && key != paxGNUSparseOffset) || - (len(sparseMap)%2 == 1 && key != paxGNUSparseNumBytes) || - strings.Contains(value, ",") { - return nil, ErrHeader - } - sparseMap = append(sparseMap, value) - default: - paxHdrs[key] = value - } - } - if len(sparseMap) > 0 { - paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") - } - return paxHdrs, nil -} - -// readHeader reads the next block header and assumes that the underlying reader -// is already aligned to a block boundary. It returns the raw block of the -// header in case further processing is required. -// -// The err will be set to io.EOF only when one of the following occurs: -// - Exactly 0 bytes are read and EOF is hit. -// - Exactly 1 block of zeros is read and EOF is hit. -// - At least 2 blocks of zeros are read. -func (tr *Reader) readHeader() (*Header, *block, error) { - // Two blocks of zero bytes marks the end of the archive. - n, err := io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 0 bytes read - } - - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - n, err = io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 1 block of zeros read - } - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - return nil, nil, io.EOF // normal EOF; exactly 2 block of zeros read - } - return nil, nil, ErrHeader // Zero block and then non-zero block - } - - // Verify the header matches a known format. - format := tr.blk.GetFormat() - if format == FormatUnknown { - return nil, nil, ErrHeader - } - - var p parser - hdr := new(Header) - - // Unpack the V7 header. - v7 := tr.blk.V7() - hdr.Typeflag = v7.TypeFlag()[0] - hdr.Name = p.parseString(v7.Name()) - hdr.Linkname = p.parseString(v7.LinkName()) - hdr.Size = p.parseNumeric(v7.Size()) - hdr.Mode = p.parseNumeric(v7.Mode()) - hdr.Uid = int(p.parseNumeric(v7.UID())) - hdr.Gid = int(p.parseNumeric(v7.GID())) - hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0) - - // Unpack format specific fields. - if format > formatV7 { - ustar := tr.blk.USTAR() - hdr.Uname = p.parseString(ustar.UserName()) - hdr.Gname = p.parseString(ustar.GroupName()) - hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) - hdr.Devminor = p.parseNumeric(ustar.DevMinor()) - - var prefix string - switch { - case format.has(FormatUSTAR | FormatPAX): - hdr.Format = format - ustar := tr.blk.USTAR() - prefix = p.parseString(ustar.Prefix()) - - // For Format detection, check if block is properly formatted since - // the parser is more liberal than what USTAR actually permits. - notASCII := func(r rune) bool { return r >= 0x80 } - if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 { - hdr.Format = FormatUnknown // Non-ASCII characters in block. - } - nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 } - if !(nul(v7.Size()) && nul(v7.Mode()) && nul(v7.UID()) && nul(v7.GID()) && - nul(v7.ModTime()) && nul(ustar.DevMajor()) && nul(ustar.DevMinor())) { - hdr.Format = FormatUnknown // Numeric fields must end in NUL - } - case format.has(formatSTAR): - star := tr.blk.STAR() - prefix = p.parseString(star.Prefix()) - hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0) - hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0) - case format.has(FormatGNU): - hdr.Format = format - var p2 parser - gnu := tr.blk.GNU() - if b := gnu.AccessTime(); b[0] != 0 { - hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0) - } - if b := gnu.ChangeTime(); b[0] != 0 { - hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0) - } - - // Prior to Go1.8, the Writer had a bug where it would output - // an invalid tar file in certain rare situations because the logic - // incorrectly believed that the old GNU format had a prefix field. - // This is wrong and leads to an output file that mangles the - // atime and ctime fields, which are often left unused. - // - // In order to continue reading tar files created by former, buggy - // versions of Go, we skeptically parse the atime and ctime fields. - // If we are unable to parse them and the prefix field looks like - // an ASCII string, then we fallback on the pre-Go1.8 behavior - // of treating these fields as the USTAR prefix field. - // - // Note that this will not use the fallback logic for all possible - // files generated by a pre-Go1.8 toolchain. If the generated file - // happened to have a prefix field that parses as valid - // atime and ctime fields (e.g., when they are valid octal strings), - // then it is impossible to distinguish between an valid GNU file - // and an invalid pre-Go1.8 file. - // - // See https://golang.org/issues/12594 - // See https://golang.org/issues/21005 - if p2.err != nil { - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - ustar := tr.blk.USTAR() - if s := p.parseString(ustar.Prefix()); isASCII(s) { - prefix = s - } - hdr.Format = FormatUnknown // Buggy file is not GNU - } - } - if len(prefix) > 0 { - hdr.Name = prefix + "/" + hdr.Name - } - } - return hdr, &tr.blk, p.err -} - -// readOldGNUSparseMap reads the sparse map from the old GNU sparse format. -// The sparse map is stored in the tar header if it's small enough. -// If it's larger than four entries, then one or more extension headers are used -// to store the rest of the sparse map. -// -// The Header.Size does not reflect the size of any extended headers used. -// Thus, this function will read from the raw io.Reader to fetch extra headers. -// This method mutates blk in the process. -func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) { - // Make sure that the input format is GNU. - // Unfortunately, the STAR format also has a sparse header format that uses - // the same type flag but has a completely different layout. - if blk.GetFormat() != FormatGNU { - return nil, ErrHeader - } - hdr.Format.mayOnlyBe(FormatGNU) - - var p parser - hdr.Size = p.parseNumeric(blk.GNU().RealSize()) - if p.err != nil { - return nil, p.err - } - s := blk.GNU().Sparse() - spd := make(sparseDatas, 0, s.MaxEntries()) - for { - for i := 0; i < s.MaxEntries(); i++ { - // This termination condition is identical to GNU and BSD tar. - if s.Entry(i).Offset()[0] == 0x00 { - break // Don't return, need to process extended headers (even if empty) - } - offset := p.parseNumeric(s.Entry(i).Offset()) - length := p.parseNumeric(s.Entry(i).Length()) - if p.err != nil { - return nil, p.err - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - - if s.IsExtended()[0] > 0 { - // There are more entries. Read an extension header and parse its entries. - if _, err := mustReadFull(tr.r, blk[:]); err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(blk[:]) - } - s = blk.Sparse() - continue - } - return spd, nil // Done - } -} - -// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format -// version 1.0. The format of the sparse map consists of a series of -// newline-terminated numeric fields. The first field is the number of entries -// and is always present. Following this are the entries, consisting of two -// fields (offset, length). This function must stop reading at the end -// boundary of the block containing the last newline. -// -// Note that the GNU manual says that numeric values should be encoded in octal -// format. However, the GNU tar utility itself outputs these values in decimal. -// As such, this library treats values as being encoded in decimal. -func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) { - var ( - cntNewline int64 - buf bytes.Buffer - blk block - totalSize int - ) - - // feedTokens copies data in blocks from r into buf until there are - // at least cnt newlines in buf. It will not read more blocks than needed. - feedTokens := func(n int64) error { - for cntNewline < n { - totalSize += len(blk) - if totalSize > maxSpecialFileSize { - return errSparseTooLong - } - if _, err := mustReadFull(r, blk[:]); err != nil { - return err - } - buf.Write(blk[:]) - for _, c := range blk { - if c == '\n' { - cntNewline++ - } - } - } - return nil - } - - // nextToken gets the next token delimited by a newline. This assumes that - // at least one newline exists in the buffer. - nextToken := func() string { - cntNewline-- - tok, _ := buf.ReadString('\n') - return strings.TrimRight(tok, "\n") - } - - // Parse for the number of entries. - // Use integer overflow resistant math to check this. - if err := feedTokens(1); err != nil { - return nil, err - } - numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // Parse for all member entries. - // numEntries is trusted after this since feedTokens limits the number of - // tokens based on maxSpecialFileSize. - if err := feedTokens(2 * numEntries); err != nil { - return nil, err - } - spd := make(sparseDatas, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - offset, err1 := strconv.ParseInt(nextToken(), 10, 64) - length, err2 := strconv.ParseInt(nextToken(), 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - return spd, nil -} - -// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format -// version 0.1. The sparse map is stored in the PAX headers. -func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) { - // Get number of entries. - // Use integer overflow resistant math to check this. - numEntriesStr := paxHdrs[paxGNUSparseNumBlocks] - numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // There should be two numbers in sparseMap for each entry. - sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",") - if len(sparseMap) == 1 && sparseMap[0] == "" { - sparseMap = sparseMap[:0] - } - if int64(len(sparseMap)) != 2*numEntries { - return nil, ErrHeader - } - - // Loop through the entries in the sparse map. - // numEntries is trusted now. - spd := make(sparseDatas, 0, numEntries) - for len(sparseMap) >= 2 { - offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64) - length, err2 := strconv.ParseInt(sparseMap[1], 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - sparseMap = sparseMap[2:] - } - return spd, nil -} - -// Read reads from the current file in the tar archive. -// It returns (0, io.EOF) when it reaches the end of that file, -// until Next is called to advance to the next file. -// -// If the current file is sparse, then the regions marked as a hole -// are read back as NUL-bytes. -// -// Calling Read on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what -// the Header.Size claims. -func (tr *Reader) Read(b []byte) (int, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.Read(b) - if err != nil && err != io.EOF { - tr.err = err - } - return n, err -} - -// writeTo writes the content of the current file to w. -// The bytes written matches the number of remaining bytes in the current file. -// -// If the current file is sparse and w is an io.WriteSeeker, -// then writeTo uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are filled with NULs. -// This always writes the last byte to ensure w is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tr *Reader) writeTo(w io.Writer) (int64, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.WriteTo(w) - if err != nil { - tr.err = err - } - return n, err -} - -// regFileReader is a fileReader for reading data from a regular file entry. -type regFileReader struct { - r io.Reader // Underlying Reader - nb int64 // Number of remaining bytes to read -} - -func (fr *regFileReader) Read(b []byte) (n int, err error) { - if int64(len(b)) > fr.nb { - b = b[:fr.nb] - } - if len(b) > 0 { - n, err = fr.r.Read(b) - fr.nb -= int64(n) - } - switch { - case err == io.EOF && fr.nb > 0: - return n, io.ErrUnexpectedEOF - case err == nil && fr.nb == 0: - return n, io.EOF - default: - return n, err - } -} - -func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { - return io.Copy(w, struct{ io.Reader }{fr}) -} - -func (fr regFileReader) LogicalRemaining() int64 { - return fr.nb -} - -func (fr regFileReader) PhysicalRemaining() int64 { - return fr.nb -} - -// sparseFileReader is a fileReader for reading data from a sparse file entry. -type sparseFileReader struct { - fr fileReader // Underlying fileReader - sp sparseHoles // Normalized list of sparse holes - pos int64 // Current position in sparse file -} - -func (sr *sparseFileReader) Read(b []byte) (n int, err error) { - finished := int64(len(b)) >= sr.LogicalRemaining() - if finished { - b = b[:sr.LogicalRemaining()] - } - - b0 := b - endPos := sr.pos + int64(len(b)) - for endPos > sr.pos && err == nil { - var nf int // Bytes read in fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - bf := b[:min(int64(len(b)), holeStart-sr.pos)] - nf, err = tryReadFull(sr.fr, bf) - } else { // In a hole fragment - bf := b[:min(int64(len(b)), holeEnd-sr.pos)] - nf, err = tryReadFull(zeroReader{}, bf) - } - b = b[nf:] - sr.pos += int64(nf) - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - case finished: - return n, io.EOF - default: - return n, nil - } -} - -func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { - ws, ok := w.(io.WriteSeeker) - if ok { - if _, err := ws.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(w, struct{ io.Reader }{sr}) - } - - var writeLastByte bool - pos0 := sr.pos - for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil { - var nf int64 // Size of fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - nf = holeStart - sr.pos - nf, err = io.CopyN(ws, sr.fr, nf) - } else { // In a hole fragment - nf = holeEnd - sr.pos - if sr.PhysicalRemaining() == 0 { - writeLastByte = true - nf-- - } - _, err = ws.Seek(nf, io.SeekCurrent) - } - sr.pos += nf - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // write a single byte to ensure the file is the right size. - if writeLastByte && err == nil { - _, err = ws.Write([]byte{0}) - sr.pos++ - } - - n = sr.pos - pos0 - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - default: - return n, nil - } -} - -func (sr sparseFileReader) LogicalRemaining() int64 { - return sr.sp[len(sr.sp)-1].endOffset() - sr.pos -} -func (sr sparseFileReader) PhysicalRemaining() int64 { - return sr.fr.PhysicalRemaining() -} - -type zeroReader struct{} - -func (zeroReader) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - -// mustReadFull is like io.ReadFull except it returns -// io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read. -func mustReadFull(r io.Reader, b []byte) (int, error) { - n, err := tryReadFull(r, b) - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return n, err -} - -// tryReadFull is like io.ReadFull except it returns -// io.EOF when it is hit before len(b) bytes are read. -func tryReadFull(r io.Reader, b []byte) (n int, err error) { - for len(b) > n && err == nil { - var nn int - nn, err = r.Read(b[n:]) - n += nn - } - if len(b) == n && err == io.EOF { - err = nil - } - return n, err -} - -// readSpecialFile is like io.ReadAll except it returns -// ErrFieldTooLong if more than maxSpecialFileSize is read. -func readSpecialFile(r io.Reader) ([]byte, error) { - buf, err := io.ReadAll(io.LimitReader(r, maxSpecialFileSize+1)) - if len(buf) > maxSpecialFileSize { - return nil, ErrFieldTooLong - } - return buf, err -} - -// discard skips n bytes in r, reporting an error if unable to do so. -func discard(tr *Reader, n int64) error { - var seekSkipped, copySkipped int64 - var err error - r := tr.r - if tr.RawAccounting { - - copySkipped, err = io.CopyN(tr.rawBytes, tr.r, n) - goto out - } - - // If possible, Seek to the last byte before the end of the data section. - // Do this because Seek is often lazy about reporting errors; this will mask - // the fact that the stream may be truncated. We can rely on the - // io.CopyN done shortly afterwards to trigger any IO errors. - if sr, ok := r.(io.Seeker); ok && n > 1 { - // Not all io.Seeker can actually Seek. For example, os.Stdin implements - // io.Seeker, but calling Seek always returns an error and performs - // no action. Thus, we try an innocent seek to the current position - // to see if Seek is really supported. - pos1, err := sr.Seek(0, io.SeekCurrent) - if pos1 >= 0 && err == nil { - // Seek seems supported, so perform the real Seek. - pos2, err := sr.Seek(n-1, io.SeekCurrent) - if pos2 < 0 || err != nil { - return err - } - seekSkipped = pos2 - pos1 - } - } - - copySkipped, err = io.CopyN(io.Discard, r, n-seekSkipped) -out: - if err == io.EOF && seekSkipped+copySkipped < n { - err = io.ErrUnexpectedEOF - } - return err -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go deleted file mode 100644 index cf9cc79c5..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux dragonfly openbsd solaris - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atim.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctim.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go deleted file mode 100644 index 6f17dbe30..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin freebsd netbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atimespec.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctimespec.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go deleted file mode 100644 index 868105f33..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux darwin dragonfly freebsd openbsd netbsd solaris - -package tar - -import ( - "os" - "os/user" - "runtime" - "strconv" - "sync" - "syscall" -) - -func init() { - sysStat = statUnix -} - -// userMap and groupMap caches UID and GID lookups for performance reasons. -// The downside is that renaming uname or gname by the OS never takes effect. -var userMap, groupMap sync.Map // map[int]string - -func statUnix(fi os.FileInfo, h *Header) error { - sys, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return nil - } - h.Uid = int(sys.Uid) - h.Gid = int(sys.Gid) - - // Best effort at populating Uname and Gname. - // The os/user functions may fail for any number of reasons - // (not implemented on that platform, cgo not enabled, etc). - if u, ok := userMap.Load(h.Uid); ok { - h.Uname = u.(string) - } else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil { - h.Uname = u.Username - userMap.Store(h.Uid, h.Uname) - } - if g, ok := groupMap.Load(h.Gid); ok { - h.Gname = g.(string) - } else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil { - h.Gname = g.Name - groupMap.Store(h.Gid, h.Gname) - } - - h.AccessTime = statAtime(sys) - h.ChangeTime = statCtime(sys) - - // Best effort at populating Devmajor and Devminor. - if h.Typeflag == TypeChar || h.Typeflag == TypeBlock { - dev := uint64(sys.Rdev) // May be int32 or uint32 - switch runtime.GOOS { - case "linux": - // Copied from golang.org/x/sys/unix/dev_linux.go. - major := uint32((dev & 0x00000000000fff00) >> 8) - major |= uint32((dev & 0xfffff00000000000) >> 32) - minor := uint32((dev & 0x00000000000000ff) >> 0) - minor |= uint32((dev & 0x00000ffffff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "darwin": - // Copied from golang.org/x/sys/unix/dev_darwin.go. - major := uint32((dev >> 24) & 0xff) - minor := uint32(dev & 0xffffff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "dragonfly": - // Copied from golang.org/x/sys/unix/dev_dragonfly.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "freebsd": - // Copied from golang.org/x/sys/unix/dev_freebsd.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "netbsd": - // Copied from golang.org/x/sys/unix/dev_netbsd.go. - major := uint32((dev & 0x000fff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xfff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "openbsd": - // Copied from golang.org/x/sys/unix/dev_openbsd.go. - major := uint32((dev & 0x0000ff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xffff0000) >> 8) - h.Devmajor, h.Devminor = int64(major), int64(minor) - default: - // TODO: Implement solaris (see https://golang.org/issue/8106) - } - } - return nil -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go b/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go deleted file mode 100644 index d144485a4..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go +++ /dev/null @@ -1,326 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "time" -) - -// hasNUL reports whether the NUL character exists within s. -func hasNUL(s string) bool { - return strings.IndexByte(s, 0) >= 0 -} - -// isASCII reports whether the input is an ASCII C-style string. -func isASCII(s string) bool { - for _, c := range s { - if c >= 0x80 || c == 0x00 { - return false - } - } - return true -} - -// toASCII converts the input to an ASCII C-style string. -// This a best effort conversion, so invalid characters are dropped. -func toASCII(s string) string { - if isASCII(s) { - return s - } - b := make([]byte, 0, len(s)) - for _, c := range s { - if c < 0x80 && c != 0x00 { - b = append(b, byte(c)) - } - } - return string(b) -} - -type parser struct { - err error // Last error seen -} - -type formatter struct { - err error // Last error seen -} - -// parseString parses bytes as a NUL-terminated C-style string. -// If a NUL byte is not found then the whole slice is returned as a string. -func (*parser) parseString(b []byte) string { - if i := bytes.IndexByte(b, 0); i >= 0 { - return string(b[:i]) - } - return string(b) -} - -// formatString copies s into b, NUL-terminating if possible. -func (f *formatter) formatString(b []byte, s string) { - if len(s) > len(b) { - f.err = ErrFieldTooLong - } - copy(b, s) - if len(s) < len(b) { - b[len(s)] = 0 - } - - // Some buggy readers treat regular files with a trailing slash - // in the V7 path field as a directory even though the full path - // recorded elsewhere (e.g., via PAX record) contains no trailing slash. - if len(s) > len(b) && b[len(b)-1] == '/' { - n := len(strings.TrimRight(s[:len(b)], "/")) - b[n] = 0 // Replace trailing slash with NUL terminator - } -} - -// fitsInBase256 reports whether x can be encoded into n bytes using base-256 -// encoding. Unlike octal encoding, base-256 encoding does not require that the -// string ends with a NUL character. Thus, all n bytes are available for output. -// -// If operating in binary mode, this assumes strict GNU binary mode; which means -// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is -// equivalent to the sign bit in two's complement form. -func fitsInBase256(n int, x int64) bool { - binBits := uint(n-1) * 8 - return n >= 9 || (x >= -1< 0 && b[0]&0x80 != 0 { - // Handling negative numbers relies on the following identity: - // -a-1 == ^a - // - // If the number is negative, we use an inversion mask to invert the - // data bytes and treat the value as an unsigned number. - var inv byte // 0x00 if positive or zero, 0xff if negative - if b[0]&0x40 != 0 { - inv = 0xff - } - - var x uint64 - for i, c := range b { - c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing - if i == 0 { - c &= 0x7f // Ignore signal bit in first byte - } - if (x >> 56) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - x = x<<8 | uint64(c) - } - if (x >> 63) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - if inv == 0xff { - return ^int64(x) - } - return int64(x) - } - - // Normal case is base-8 (octal) format. - return p.parseOctal(b) -} - -// formatNumeric encodes x into b using base-8 (octal) encoding if possible. -// Otherwise it will attempt to use base-256 (binary) encoding. -func (f *formatter) formatNumeric(b []byte, x int64) { - if fitsInOctal(len(b), x) { - f.formatOctal(b, x) - return - } - - if fitsInBase256(len(b), x) { - for i := len(b) - 1; i >= 0; i-- { - b[i] = byte(x) - x >>= 8 - } - b[0] |= 0x80 // Highest bit indicates binary format - return - } - - f.formatOctal(b, 0) // Last resort, just write zero - f.err = ErrFieldTooLong -} - -func (p *parser) parseOctal(b []byte) int64 { - // Because unused fields are filled with NULs, we need - // to skip leading NULs. Fields may also be padded with - // spaces or NULs. - // So we remove leading and trailing NULs and spaces to - // be sure. - b = bytes.Trim(b, " \x00") - - if len(b) == 0 { - return 0 - } - x, perr := strconv.ParseUint(p.parseString(b), 8, 64) - if perr != nil { - p.err = ErrHeader - } - return int64(x) -} - -func (f *formatter) formatOctal(b []byte, x int64) { - if !fitsInOctal(len(b), x) { - x = 0 // Last resort, just write zero - f.err = ErrFieldTooLong - } - - s := strconv.FormatInt(x, 8) - // Add leading zeros, but leave room for a NUL. - if n := len(b) - len(s) - 1; n > 0 { - s = strings.Repeat("0", n) + s - } - f.formatString(b, s) -} - -// fitsInOctal reports whether the integer x fits in a field n-bytes long -// using octal encoding with the appropriate NUL terminator. -func fitsInOctal(n int, x int64) bool { - octBits := uint(n-1) * 3 - return x >= 0 && (n >= 22 || x < 1<= 0 { - ss, sn = s[:pos], s[pos+1:] - } - - // Parse the seconds. - secs, err := strconv.ParseInt(ss, 10, 64) - if err != nil { - return time.Time{}, ErrHeader - } - if len(sn) == 0 { - return time.Unix(secs, 0), nil // No sub-second values - } - - // Parse the nanoseconds. - if strings.Trim(sn, "0123456789") != "" { - return time.Time{}, ErrHeader - } - if len(sn) < maxNanoSecondDigits { - sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad - } else { - sn = sn[:maxNanoSecondDigits] // Right truncate - } - nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed - if len(ss) > 0 && ss[0] == '-' { - return time.Unix(secs, -1*nsecs), nil // Negative correction - } - return time.Unix(secs, nsecs), nil -} - -// formatPAXTime converts ts into a time of the form %d.%d as described in the -// PAX specification. This function is capable of negative timestamps. -func formatPAXTime(ts time.Time) (s string) { - secs, nsecs := ts.Unix(), ts.Nanosecond() - if nsecs == 0 { - return strconv.FormatInt(secs, 10) - } - - // If seconds is negative, then perform correction. - sign := "" - if secs < 0 { - sign = "-" // Remember sign - secs = -(secs + 1) // Add a second to secs - nsecs = -(nsecs - 1E9) // Take that second away from nsecs - } - return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0") -} - -// parsePAXRecord parses the input PAX record string into a key-value pair. -// If parsing is successful, it will slice off the currently read record and -// return the remainder as r. -func parsePAXRecord(s string) (k, v, r string, err error) { - // The size field ends at the first space. - sp := strings.IndexByte(s, ' ') - if sp == -1 { - return "", "", s, ErrHeader - } - - // Parse the first token as a decimal integer. - n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int - if perr != nil || n < 5 || int64(len(s)) < n { - return "", "", s, ErrHeader - } - - // Extract everything between the space and the final newline. - rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:] - if nl != "\n" { - return "", "", s, ErrHeader - } - - // The first equals separates the key from the value. - eq := strings.IndexByte(rec, '=') - if eq == -1 { - return "", "", s, ErrHeader - } - k, v = rec[:eq], rec[eq+1:] - - if !validPAXRecord(k, v) { - return "", "", s, ErrHeader - } - return k, v, rem, nil -} - -// formatPAXRecord formats a single PAX record, prefixing it with the -// appropriate length. -func formatPAXRecord(k, v string) (string, error) { - if !validPAXRecord(k, v) { - return "", ErrHeader - } - - const padding = 3 // Extra padding for ' ', '=', and '\n' - size := len(k) + len(v) + padding - size += len(strconv.Itoa(size)) - record := strconv.Itoa(size) + " " + k + "=" + v + "\n" - - // Final adjustment if adding size field increased the record size. - if len(record) != size { - size = len(record) - record = strconv.Itoa(size) + " " + k + "=" + v + "\n" - } - return record, nil -} - -// validPAXRecord reports whether the key-value pair is valid where each -// record is formatted as: -// "%d %s=%s\n" % (size, key, value) -// -// Keys and values should be UTF-8, but the number of bad writers out there -// forces us to be a more liberal. -// Thus, we only reject all keys with NUL, and only reject NULs in values -// for the PAX version of the USTAR string fields. -// The key must not contain an '=' character. -func validPAXRecord(k, v string) bool { - if k == "" || strings.IndexByte(k, '=') >= 0 { - return false - } - switch k { - case paxPath, paxLinkpath, paxUname, paxGname: - return !hasNUL(v) - default: - return !hasNUL(k) - } -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go b/vendor/github.com/vbatts/tar-split/archive/tar/writer.go deleted file mode 100644 index 893eac00a..000000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go +++ /dev/null @@ -1,656 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "fmt" - "io" - "path" - "sort" - "strings" - "time" -) - -// Writer provides sequential writing of a tar archive. -// Write.WriteHeader begins a new file with the provided Header, -// and then Writer can be treated as an io.Writer to supply that file's data. -type Writer struct { - w io.Writer - pad int64 // Amount of padding to write after current file entry - curr fileWriter // Writer for current file entry - hdr Header // Shallow copy of Header that is safe for mutations - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Writer to - // ensure that this error is sticky. - err error -} - -// NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { - return &Writer{w: w, curr: ®FileWriter{w, 0}} -} - -type fileWriter interface { - io.Writer - fileState - - ReadFrom(io.Reader) (int64, error) -} - -// Flush finishes writing the current file's block padding. -// The current file must be fully written before Flush can be called. -// -// This is unnecessary as the next call to WriteHeader or Close -// will implicitly flush out the file's padding. -func (tw *Writer) Flush() error { - if tw.err != nil { - return tw.err - } - if nb := tw.curr.LogicalRemaining(); nb > 0 { - return fmt.Errorf("archive/tar: missed writing %d bytes", nb) - } - if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil { - return tw.err - } - tw.pad = 0 - return nil -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// The Header.Size determines how many bytes can be written for the next file. -// If the current file is not fully written, then this returns an error. -// This implicitly flushes any padding necessary before writing the header. -func (tw *Writer) WriteHeader(hdr *Header) error { - if err := tw.Flush(); err != nil { - return err - } - tw.hdr = *hdr // Shallow copy of Header - - // Avoid usage of the legacy TypeRegA flag, and automatically promote - // it to use TypeReg or TypeDir. - if tw.hdr.Typeflag == TypeRegA { - if strings.HasSuffix(tw.hdr.Name, "/") { - tw.hdr.Typeflag = TypeDir - } else { - tw.hdr.Typeflag = TypeReg - } - } - - // Round ModTime and ignore AccessTime and ChangeTime unless - // the format is explicitly chosen. - // This ensures nominal usage of WriteHeader (without specifying the format) - // does not always result in the PAX format being chosen, which - // causes a 1KiB increase to every header. - if tw.hdr.Format == FormatUnknown { - tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second) - tw.hdr.AccessTime = time.Time{} - tw.hdr.ChangeTime = time.Time{} - } - - allowedFormats, paxHdrs, err := tw.hdr.allowedFormats() - switch { - case allowedFormats.has(FormatUSTAR): - tw.err = tw.writeUSTARHeader(&tw.hdr) - return tw.err - case allowedFormats.has(FormatPAX): - tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs) - return tw.err - case allowedFormats.has(FormatGNU): - tw.err = tw.writeGNUHeader(&tw.hdr) - return tw.err - default: - return err // Non-fatal error - } -} - -func (tw *Writer) writeUSTARHeader(hdr *Header) error { - // Check if we can use USTAR prefix/suffix splitting. - var namePrefix string - if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok { - namePrefix, hdr.Name = prefix, suffix - } - - // Pack the main header. - var f formatter - blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal) - f.formatString(blk.USTAR().Prefix(), namePrefix) - blk.SetFormat(FormatUSTAR) - if f.err != nil { - return f.err // Should never happen since header is validated - } - return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag) -} - -func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { - realName, realSize := hdr.Name, hdr.Size - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Handle sparse files. - var spd sparseDatas - var spb []byte - if len(hdr.SparseHoles) > 0 { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - hdr.Size = 0 // Replace with encoded size - spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n') - for _, s := range spd { - hdr.Size += s.Length - spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n') - spb = append(strconv.AppendInt(spb, s.Length, 10), '\n') - } - pad := blockPadding(int64(len(spb))) - spb = append(spb, zeroBlock[:pad]...) - hdr.Size += int64(len(spb)) // Accounts for encoded sparse map - - // Add and modify appropriate PAX records. - dir, file := path.Split(realName) - hdr.Name = path.Join(dir, "GNUSparseFile.0", file) - paxHdrs[paxGNUSparseMajor] = "1" - paxHdrs[paxGNUSparseMinor] = "0" - paxHdrs[paxGNUSparseName] = realName - paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10) - paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10) - delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName - } - */ - _ = realSize - - // Write PAX records to the output. - isGlobal := hdr.Typeflag == TypeXGlobalHeader - if len(paxHdrs) > 0 || isGlobal { - // Sort keys for deterministic ordering. - var keys []string - for k := range paxHdrs { - keys = append(keys, k) - } - sort.Strings(keys) - - // Write each record to a buffer. - var buf strings.Builder - for _, k := range keys { - rec, err := formatPAXRecord(k, paxHdrs[k]) - if err != nil { - return err - } - buf.WriteString(rec) - } - - // Write the extended header file. - var name string - var flag byte - if isGlobal { - name = realName - if name == "" { - name = "GlobalHead.0.0" - } - flag = TypeXGlobalHeader - } else { - dir, file := path.Split(realName) - name = path.Join(dir, "PaxHeaders.0", file) - flag = TypeXHeader - } - data := buf.String() - if len(data) > maxSpecialFileSize { - return ErrFieldTooLong - } - if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal { - return err // Global headers return here - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) } - blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal) - blk.SetFormat(FormatPAX) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Write the sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.curr since the sparse map is accounted for in hdr.Size. - if _, err := tw.curr.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - */ - return nil -} - -func (tw *Writer) writeGNUHeader(hdr *Header) error { - // Use long-link files if Name or Linkname exceeds the field size. - const longName = "././@LongLink" - if len(hdr.Name) > nameSize { - data := hdr.Name + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil { - return err - } - } - if len(hdr.Linkname) > nameSize { - data := hdr.Linkname + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil { - return err - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - var spd sparseDatas - var spb []byte - blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric) - if !hdr.AccessTime.IsZero() { - f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix()) - } - if !hdr.ChangeTime.IsZero() { - f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix()) - } - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - if hdr.Typeflag == TypeGNUSparse { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas { - for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ { - f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset) - f.formatNumeric(sa.Entry(i).Length(), sp[0].Length) - sp = sp[1:] - } - if len(sp) > 0 { - sa.IsExtended()[0] = 1 - } - return sp - } - sp2 := formatSPD(spd, blk.GNU().Sparse()) - for len(sp2) > 0 { - var spHdr block - sp2 = formatSPD(sp2, spHdr.Sparse()) - spb = append(spb, spHdr[:]...) - } - - // Update size fields in the header block. - realSize := hdr.Size - hdr.Size = 0 // Encoded size; does not account for encoded sparse map - for _, s := range spd { - hdr.Size += s.Length - } - copy(blk.V7().Size(), zeroBlock[:]) // Reset field - f.formatNumeric(blk.V7().Size(), hdr.Size) - f.formatNumeric(blk.GNU().RealSize(), realSize) - } - */ - blk.SetFormat(FormatGNU) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // Write the extended sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.w since the sparse map is not accounted for in hdr.Size. - if _, err := tw.w.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - return nil -} - -type ( - stringFormatter func([]byte, string) - numberFormatter func([]byte, int64) -) - -// templateV7Plus fills out the V7 fields of a block using values from hdr. -// It also fills out fields (uname, gname, devmajor, devminor) that are -// shared in the USTAR, PAX, and GNU formats using the provided formatters. -// -// The block returned is only valid until the next call to -// templateV7Plus or writeRawFile. -func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block { - tw.blk.Reset() - - modTime := hdr.ModTime - if modTime.IsZero() { - modTime = time.Unix(0, 0) - } - - v7 := tw.blk.V7() - v7.TypeFlag()[0] = hdr.Typeflag - fmtStr(v7.Name(), hdr.Name) - fmtStr(v7.LinkName(), hdr.Linkname) - fmtNum(v7.Mode(), hdr.Mode) - fmtNum(v7.UID(), int64(hdr.Uid)) - fmtNum(v7.GID(), int64(hdr.Gid)) - fmtNum(v7.Size(), hdr.Size) - fmtNum(v7.ModTime(), modTime.Unix()) - - ustar := tw.blk.USTAR() - fmtStr(ustar.UserName(), hdr.Uname) - fmtStr(ustar.GroupName(), hdr.Gname) - fmtNum(ustar.DevMajor(), hdr.Devmajor) - fmtNum(ustar.DevMinor(), hdr.Devminor) - - return &tw.blk -} - -// writeRawFile writes a minimal file with the given name and flag type. -// It uses format to encode the header format and will write data as the body. -// It uses default values for all of the other fields (as BSD and GNU tar does). -func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error { - tw.blk.Reset() - - // Best effort for the filename. - name = toASCII(name) - if len(name) > nameSize { - name = name[:nameSize] - } - name = strings.TrimRight(name, "/") - - var f formatter - v7 := tw.blk.V7() - v7.TypeFlag()[0] = flag - f.formatString(v7.Name(), name) - f.formatOctal(v7.Mode(), 0) - f.formatOctal(v7.UID(), 0) - f.formatOctal(v7.GID(), 0) - f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB - f.formatOctal(v7.ModTime(), 0) - tw.blk.SetFormat(format) - if f.err != nil { - return f.err // Only occurs if size condition is violated - } - - // Write the header and data. - if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil { - return err - } - _, err := io.WriteString(tw, data) - return err -} - -// writeRawHeader writes the value of blk, regardless of its value. -// It sets up the Writer such that it can accept a file of the given size. -// If the flag is a special header-only flag, then the size is treated as zero. -func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error { - if err := tw.Flush(); err != nil { - return err - } - if _, err := tw.w.Write(blk[:]); err != nil { - return err - } - if isHeaderOnlyType(flag) { - size = 0 - } - tw.curr = ®FileWriter{tw.w, size} - tw.pad = blockPadding(size) - return nil -} - -// splitUSTARPath splits a path according to USTAR prefix and suffix rules. -// If the path is not splittable, then it will return ("", "", false). -func splitUSTARPath(name string) (prefix, suffix string, ok bool) { - length := len(name) - if length <= nameSize || !isASCII(name) { - return "", "", false - } else if length > prefixSize+1 { - length = prefixSize + 1 - } else if name[length-1] == '/' { - length-- - } - - i := strings.LastIndex(name[:length], "/") - nlen := len(name) - i - 1 // nlen is length of suffix - plen := i // plen is length of prefix - if i <= 0 || nlen > nameSize || nlen == 0 || plen > prefixSize { - return "", "", false - } - return name[:i], name[i+1:], true -} - -// Write writes to the current file in the tar archive. -// Write returns the error ErrWriteTooLong if more than -// Header.Size bytes are written after WriteHeader. -// -// Calling Write on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless -// of what the Header.Size claims. -func (tw *Writer) Write(b []byte) (int, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.Write(b) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// readFrom populates the content of the current file by reading from r. -// The bytes read must match the number of remaining bytes in the current file. -// -// If the current file is sparse and r is an io.ReadSeeker, -// then readFrom uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are all NULs. -// This always reads the last byte to ensure r is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tw *Writer) readFrom(r io.Reader) (int64, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.ReadFrom(r) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// Close closes the tar archive by flushing the padding, and writing the footer. -// If the current file (from a prior call to WriteHeader) is not fully written, -// then this returns an error. -func (tw *Writer) Close() error { - if tw.err == ErrWriteAfterClose { - return nil - } - if tw.err != nil { - return tw.err - } - - // Trailer: two zero blocks. - err := tw.Flush() - for i := 0; i < 2 && err == nil; i++ { - _, err = tw.w.Write(zeroBlock[:]) - } - - // Ensure all future actions are invalid. - tw.err = ErrWriteAfterClose - return err // Report IO errors -} - -// regFileWriter is a fileWriter for writing data to a regular file entry. -type regFileWriter struct { - w io.Writer // Underlying Writer - nb int64 // Number of remaining bytes to write -} - -func (fw *regFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > fw.nb - if overwrite { - b = b[:fw.nb] - } - if len(b) > 0 { - n, err = fw.w.Write(b) - fw.nb -= int64(n) - } - switch { - case err != nil: - return n, err - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) { - return io.Copy(struct{ io.Writer }{fw}, r) -} - -func (fw regFileWriter) LogicalRemaining() int64 { - return fw.nb -} -func (fw regFileWriter) PhysicalRemaining() int64 { - return fw.nb -} - -// sparseFileWriter is a fileWriter for writing data to a sparse file entry. -type sparseFileWriter struct { - fw fileWriter // Underlying fileWriter - sp sparseDatas // Normalized list of data fragments - pos int64 // Current position in sparse file -} - -func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > sw.LogicalRemaining() - if overwrite { - b = b[:sw.LogicalRemaining()] - } - - b0 := b - endPos := sw.pos + int64(len(b)) - for endPos > sw.pos && err == nil { - var nf int // Bytes written in fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - bf := b[:min(int64(len(b)), dataStart-sw.pos)] - nf, err = zeroWriter{}.Write(bf) - } else { // In a data fragment - bf := b[:min(int64(len(b)), dataEnd-sw.pos)] - nf, err = sw.fw.Write(bf) - } - b = b[nf:] - sw.pos += int64(nf) - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) { - rs, ok := r.(io.ReadSeeker) - if ok { - if _, err := rs.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(struct{ io.Writer }{sw}, r) - } - - var readLastByte bool - pos0 := sw.pos - for sw.LogicalRemaining() > 0 && !readLastByte && err == nil { - var nf int64 // Size of fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - nf = dataStart - sw.pos - if sw.PhysicalRemaining() == 0 { - readLastByte = true - nf-- - } - _, err = rs.Seek(nf, io.SeekCurrent) - } else { // In a data fragment - nf = dataEnd - sw.pos - nf, err = io.CopyN(sw.fw, rs, nf) - } - sw.pos += nf - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // read a single byte to ensure the file is the right size. - if readLastByte && err == nil { - _, err = mustReadFull(rs, []byte{0}) - sw.pos++ - } - - n = sw.pos - pos0 - switch { - case err == io.EOF: - return n, io.ErrUnexpectedEOF - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - default: - return n, ensureEOF(rs) - } -} - -func (sw sparseFileWriter) LogicalRemaining() int64 { - return sw.sp[len(sw.sp)-1].endOffset() - sw.pos -} -func (sw sparseFileWriter) PhysicalRemaining() int64 { - return sw.fw.PhysicalRemaining() -} - -// zeroWriter may only be written with NULs, otherwise it returns errWriteHole. -type zeroWriter struct{} - -func (zeroWriter) Write(b []byte) (int, error) { - for i, c := range b { - if c != 0 { - return i, errWriteHole - } - } - return len(b), nil -} - -// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so. -func ensureEOF(r io.Reader) error { - n, err := tryReadFull(r, []byte{0}) - switch { - case n > 0: - return ErrWriteTooLong - case err == io.EOF: - return nil - default: - return err - } -} diff --git a/vendor/go.opentelemetry.io/otel/.golangci.yml b/vendor/go.opentelemetry.io/otel/.golangci.yml index db1f55101..645c7e6af 100644 --- a/vendor/go.opentelemetry.io/otel/.golangci.yml +++ b/vendor/go.opentelemetry.io/otel/.golangci.yml @@ -96,9 +96,9 @@ linters: - "!**/exporters/zipkin/**" deny: - pkg: go.opentelemetry.io/otel/semconv - desc: "Use go.opentelemetry.io/otel/semconv/v1.40.0 instead. If a newer semconv version has been released, update the depguard rule." + desc: "Use go.opentelemetry.io/otel/semconv/v1.41.0 instead. If a newer semconv version has been released, update the depguard rule." allow: - - go.opentelemetry.io/otel/semconv/v1.40.0 + - go.opentelemetry.io/otel/semconv/v1.41.0 gocritic: disabled-checks: - appendAssign @@ -134,13 +134,16 @@ linters: strconcat: true revive: confidence: 0.01 + enable-all-rules: false + enable-default-rules: true + max-open-files: 2048 rules: - name: blank-imports - name: bool-literal-in-expr - name: constant-logical-expr - name: context-as-argument arguments: - - allowTypesBefore: '*testing.T' + - allow-types-before: '*testing.T' disabled: true - name: context-keys-type - name: deep-exit @@ -152,7 +155,7 @@ linters: - name: duplicated-imports - name: early-return arguments: - - preserveScope + - preserve-scope - name: empty-block - name: empty-lines - name: error-naming @@ -161,7 +164,7 @@ linters: - name: errorf - name: exported arguments: - - sayRepetitiveInsteadOfStutters + - say-repetitive-instead-of-stutters - name: flag-parameter - name: identical-branches - name: if-return @@ -169,11 +172,12 @@ linters: - name: increment-decrement - name: indent-error-flow arguments: - - preserveScope + - preserve-scope - name: package-comments - name: range - name: range-val-in-closure - name: range-val-address + - name: receiver-naming - name: redefines-builtin-id - name: string-format arguments: @@ -183,7 +187,7 @@ linters: - name: struct-tag - name: superfluous-else arguments: - - preserveScope + - preserve-scope - name: time-equal - name: unconditional-recursion - name: unexported-return diff --git a/vendor/go.opentelemetry.io/otel/AGENTS.md b/vendor/go.opentelemetry.io/otel/AGENTS.md new file mode 100644 index 000000000..26c0fc4dd --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/AGENTS.md @@ -0,0 +1,109 @@ +# Agent Guide for opentelemetry-go + +This file contains active, task-oriented instructions for autonomous and semi-autonomous coding agents working in this repository. + +Before starting any task, read `.github/copilot-instructions.md`, `CONTRIBUTING.md`, and this file. +Treat `.github/copilot-instructions.md` as global passive guidance for every task, including docs-only and review-only work. + +## Core expectations + +- Preserve OpenTelemetry specification compliance, API stability, and idiomatic Go. +- Prefer minimal, surgical changes over broad refactors or speculative cleanup. +- Read the package you are editing and match its existing naming, option types, error handling, comments, tests, and concurrency patterns. +- Keep public APIs backward compatible unless the task explicitly requires a breaking change. +- Keep telemetry resilient and loosely coupled. Do not introduce behavior that can unexpectedly interfere with host applications. +- Inspect boundaries carefully: input validation, resource limits, cancellation, shutdown, error propagation, concurrency, and memory growth. +- Prefer fail-safe behavior and explicit invariants over implicit assumptions. +- Keep dependencies minimal and justified. +- Preserve host-application safety: telemetry should not panic, block indefinitely, or amplify attacker-controlled input. +- Be conservative on hot paths. Avoid unnecessary allocations, reflection, interface churn, blocking, global state, and high-cardinality telemetry. +- Write comments only for intent, invariants, and non-obvious constraints. Do not add comments that restate the code. + +## Default workflow + +For new features and behavior changes, use this order unless the task explicitly says otherwise: + +1. Read the relevant package, its tests, and any package docs or `README.md`. +2. Add or update a failing unit test that captures the required behavior or regression. +3. Implement the smallest change that makes the test pass. +4. Refactor only after the behavior is locked in, and only if the refactor keeps the diff focused. +5. If the changed code is on a hot path or performance-sensitive, inspect existing benchmarks and run them. Add a benchmark if coverage is missing. +6. Update documentation artifacts as needed while the context is fresh. Follow the documentation and changelog conventions below for the specific updates required. +7. Run `make precommit` each time before considering the work complete. + +For docs-only, test-only, or review-only tasks, still start with the required repository guidance above, then skip the workflow steps that do not apply while keeping the same discipline around scope, verification, and repository conventions. + +## Verification + +- Use `make` as the canonical repository verification command. The default target is `precommit`. +- `make precommit` is the expected final verification step for linting, generation, README checks, module checks, and tests. +- During iteration, targeted commands are fine for fast feedback, but do not stop there if the task changes code. +- If you touch performance-sensitive code, run focused benchmarks and compare the results using `benchstat` in addition to `make`. + +## Documentation and changelog + +- Non-internal, non-test packages should have Go doc comments, usually in `doc.go`. +- Non-internal, non-test, non-documentation packages should also have a `README.md` with at least a title and a `pkg.go.dev` badge. +- Prefer examples over long code snippets in GoDoc when practical. +- Keep docs aligned with actual behavior. Do not leave stale comments, stale examples, or stale package documentation behind. +- For user-visible changes, update `CHANGELOG.md` under the appropriate `Added`, `Changed`, `Deprecated`, `Fixed`, or `Removed` section within `## [Unreleased]`. + +## Repository habits + +- Prefer focused diffs. Avoid drive-by cleanup. +- Follow existing option patterns and exported API conventions instead of inventing new abstractions. +- Generated files are checked in. If your change affects generation, keep generated output up to date. +- Prefer fast local search tools such as `rg` when exploring the repository. +- When changing behavior, make the invariants explicit in tests. + +## Personas + +### Feature Agent + +Use this persona for new behavior, new API surface, or spec-driven feature work. + +- Start with a failing unit test. +- Confirm the expected behavior against the spec, existing package behavior, and public API compatibility. +- Implement the smallest viable change. +- Update GoDoc, examples, `README.md`, and `CHANGELOG.md` when the change is user-visible. +- If the feature touches a hot path, check benchmarks and add one if the coverage is missing. + +### Refactoring Agent + +Use this persona when improving structure without intentionally changing behavior. + +- Treat behavior preservation as the default contract. +- Add or tighten tests before moving code if current behavior is not already pinned down. +- Avoid broad rewrites, clever abstractions, or package-wide cleanup unless explicitly requested. +- If a refactor touches a hot path, benchmark before and after. +- Keep API shape, semantics, concurrency guarantees, and failure modes unchanged unless the task says otherwise. + +### Test Agent + +Use this persona when adding missing coverage, reproducing bugs, or hardening regressions. + +- Reproduce the bug or missing behavior with the smallest failing test you can. +- Prefer testing public behavior and externally visible invariants. +- Add targeted regression tests before changing production code. +- Only change production code when it is required to make the tested behavior correct or testable. +- Keep tests deterministic, readable, and aligned with package patterns. + +### Performance Agent + +Use this persona for hot-path work, allocation reduction, or throughput and latency improvements. + +- Benchmark first to establish a baseline. +- Prefer changes that reduce allocations, copying, interface churn, and unnecessary synchronization. +- Do not trade away correctness, spec compliance, or API stability for micro-optimizations. +- Add or update benchmarks when performance-sensitive coverage is missing. +- If you materially change a hot path, capture before-and-after results, preferably with `benchstat`. + +### Review Agent + +Use this persona when asked to review code, patches, or pull requests. + +- Lead with findings, not summaries. +- Order findings by severity and include precise file and line references when available. +- Focus on correctness, spec compliance, API compatibility, concurrency safety, resilience, performance regressions, missing tests, missing benchmarks, documentation gaps, and changelog gaps. +- Call out when a diff is broader than necessary. +- If you find no issues, say that explicitly and note any residual risks or verification gaps. diff --git a/vendor/go.opentelemetry.io/otel/CHANGELOG.md b/vendor/go.opentelemetry.io/otel/CHANGELOG.md index 20edda441..6a90451f5 100644 --- a/vendor/go.opentelemetry.io/otel/CHANGELOG.md +++ b/vendor/go.opentelemetry.io/otel/CHANGELOG.md @@ -11,6 +11,100 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## [1.44.0/0.66.0/0.20.0/0.0.17] 2026-05-27 + +### Added + +- Add `ByteSlice` and `ByteSliceValue` functions for new `BYTESLICE` attribute type in `go.opentelemetry.io/otel/attribute`. (#7948) +- Apply attribute value limit to the `KindBytes` attribute type in `go.opentelemetry.io/otel/sdk/log`. (#7990) +- Apply attribute value limit to the `BYTESLICE` attribute type in `go.opentelemetry.io/otel/sdk/trace`. (#7990) +- Support `BYTESLICE` attributes in `go.opentelemetry.io/otel/trace`. (#8153) +- Support `BYTESLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlptrace`. (#8153) +- Support `BYTESLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlplog`. (#8153) +- Support `BYTESLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric`. (#8153) +- Support `BYTESLICE` attributes in `go.opentelemetry.io/otel/exporters/zipkin`. (#8153) +- Add `String` method for `Value` type in `go.opentelemetry.io/otel/attribute`. (#8142) +- Add `Slice` and `SliceValue` functions for new `SLICE` attribute type in `go.opentelemetry.io/otel/attribute`. (#8166) +- Support `SLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlptrace`. (#8216) +- Support `SLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlplog`. (#8216) +- Support `SLICE` attributes in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric`. (#8216) +- Support `SLICE` attributes in `go.opentelemetry.io/otel/exporters/zipkin`. (#8216) +- Apply `AttributeValueLengthLimit` to `attribute.SLICE` type attribute values in `go.opentelemetry.io/otel/sdk/trace`, recursively truncating contained string values. (#8217) +- Add `Error` field on `Record` type in `go.opentelemetry.io/otel/log/logtest`. (#8148) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc`. (#8157) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`. (#8157) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc`. (#8157) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. (#8157) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc`. (#8157) +- Add `WithMaxRequestSize` option in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. (#8157) +- Add `Settable` to `go.opentelemetry.io/otel/metric/x` to allow reusing attribute options. (#8178) +- Add experimental support for splitting metric data across multiple batches in `go.opentelemetry.io/otel/sdk/metric`. + Set `OTEL_GO_X_METRIC_EXPORT_BATCH_SIZE=` to enable for all periodic readers. + See `go.opentelemetry.io/otel/sdk/metric/internal/x` for feature documentation. (#8071) +- Add experimental self-observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc`. + Enable with `OTEL_GO_X_SELF_OBSERVABILITY=true` environment variable. + See `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x` for feature documentation. (#8192) +- Add experimental self-observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. + Enable with `OTEL_GO_X_SELF_OBSERVABILITY=true` environment variable. + See `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x` for feature documentation. (#8194) +- Add experimental self-observability metrics in `go.opentelemetry.io/otel/exporters/stdout/stdoutlog`. + Enable with `OTEL_GO_X_SELF_OBSERVABILITY=true` environment variable. + See `go.opentelemetry.io/otel/stdout/stdoutlog/internal/x` for feature documentation. (#8263) +- Add `WithDefaultAttributes` to `go.opentelemetry.io/otel/metric/x` to support setting default attributes on instruments. (#8135) +- Add `go.opentelemetry.io/otel/semconv/v1.41.0` package. + The package contains semantic conventions from the `v1.41.0` version of the OpenTelemetry Semantic Conventions. + See the [migration documentation](./semconv/v1.41.0/MIGRATION.md) for information on how to upgrade from `go.opentelemetry.io/otel/semconv/v1.40.0`. (#8324) +- Add Observable variants of instruments to `go.opentelemetry.io/otel/semconv/v1.41.0` package. (#8350) +- Generate explicit histogram bucket boundaries from weaver configuration for HTTP and RPC duration instruments in `go.opentelemetry.io/otel/semconv/v1.41.0`. (#8002) + +### Changed + +- ⚠️ **Breaking Change:** `go.opentelemetry.io/otel/sdk/metric` now applies a default cardinality limit of 2000 to comply with the Metrics SDK specification recommendation. + New attribute sets are dropped when the cardinality limit is reached. The measurement of these sets are aggregated into a special attribute set containing `attribute.Bool("otel.metric.overflow", true)`. + This can break users who relied on the previous unlimited default. + Set `WithCardinalityLimit(0)` or the deprecated `OTEL_GO_X_CARDINALITY_LIMIT=0` environment variable to preserve unlimited cardinality. + Note that support for `OTEL_GO_X_CARDINALITY_LIMIT` may be removed in a future release. (#8247) +- `ErrorType` in `go.opentelemetry.io/otel/semconv` now unwraps errors created with `fmt.Errorf` when deriving the `error.type` attribute. (#8133) +- `go.opentelemetry.io/otel/sdk/log` now unwraps error chains created with `fmt.Errorf` when deriving the `error.type` attribute from errors on log records. (#8133) +- `Set.MarshalLog` method in `go.opentelemetry.io/otel/attribute` now uses `Value.String` formatting following the [OpenTelemetry AnyValue representation for non-OTLP protocols](https://opentelemetry.io/docs/specs/otel/common/#anyvalue). (#8169) +- Optimize `go.opentelemetry.io/otel/sdk/metric` to return a drop reservoir and short-circuit `Offer` calls to the exemplar reservoir when `exemplar.AlwaysOffFilter` is configured. (#8211) (#8267) +- Optimize `go.opentelemetry.io/otel/sdk/metric` to return a drop reservoir for asynchronous instruments when `exemplar.TraceBasedFilter` is configured. (#8286) + +### Deprecated + +- Deprecate `Value.Emit` method in `go.opentelemetry.io/otel/attribute`. + Use `Value.String` instead. (#8176) + +### Fixed + +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Limit OTLP request size to 64 MiB by default in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. + The limit applies before compression, oversized requests are treated as non-retryable errors, and the limit can be configured with the new `WithMaxRequestSize` option. (#8157, #8365) +- Fix gzipped request body replay on redirect in `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp`. (#8135) +- Fix gzipped request body replay on redirect in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. (#8152) +- `go.opentelemetry.io/otel/exporters/prometheus` now uses `Value.String` formatting for label values following the [OpenTelemetry AnyValue representation for non-OTLP protocols](https://opentelemetry.io/docs/specs/otel/common/#anyvalue). (#8170) +- Propagate errors from the exporter when calling `Shutdown` on `BatchSpanProcessor` in `go.opentelemetry.io/otel/sdk/trace`. (#8197) +- Fix stale status code reporting on self-observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` and `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. (#8226) +- Fix a concurrent `Collect` data race and potential panic in `go.opentelemetry.io/otel/exporters/prometheus` when `WithResourceAsConstantLabels` option is used. (#8227) +- Fix race condition in `FixedSizeReservoir` in `go.opentelemetry.io/otel/sdk/metric/exemplar` by reverting #7447. (#8249) +- Fix `FixedSizeReservoir` in `go.opentelemetry.io/otel/sdk/metric/exemplar` to safely handle zero size. + A capacity check in the constructor initializes the reservoir safely and skips initialization for zero-cap; early returns in `Offer()` and `Collect()` ensure no-op behavior. (#8295) +- Fix counting of spans and logs in self-observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc`, `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`, `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc`, and `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. (#8254) +- Drop conflicting scope attributes named `name`, `version`, or `schema_url` from metric labels in `go.opentelemetry.io/otel/exporters/prometheus`, preserving the dedicated `otel_scope_name`, `otel_scope_version`, and `otel_scope_schema_url` labels. (#8264) +- Close schema files opened by `ParseFile` in `go.opentelemetry.io/otel/schema/v1.0` and `go.opentelemetry.io/otel/schema/v1.1`. ([GHSA-995v-fvrw-c78m](https://github.com/open-telemetry/opentelemetry-go/security/advisories/GHSA-995v-fvrw-c78m)) +- Enforce the 8192-byte baggage size limit during extraction/parsing, changing behavior when the limit is exceeded in `go.opentelemetry.io/otel/baggage` and `go.opentelemetry.io/otel/propagation`. (#8222) +- Fix `go.opentelemetry.io/otel/semconv/v1.41.0` to include `Attr*` helper methods for required attributes on observable instruments. (#8361) +- Limit baggage extraction error reporting in `go.opentelemetry.io/otel/propagation` to prevent malformed or oversized baggage headers from flooding logs. ([GHSA-5wrp-cwcj-q835](https://github.com/open-telemetry/opentelemetry-go/security/advisories/GHSA-5wrp-cwcj-q835)) + ## [1.43.0/0.65.0/0.19.0] 2026-04-02 ### Added @@ -3619,7 +3713,8 @@ It contains api and sdk for trace and meter. - CircleCI build CI manifest files. - CODEOWNERS file to track owners of this project. -[Unreleased]: https://github.com/open-telemetry/opentelemetry-go/compare/v1.43.0...HEAD +[Unreleased]: https://github.com/open-telemetry/opentelemetry-go/compare/v1.44.0...HEAD +[1.44.0/0.66.0/0.20.0/0.0.17]: https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.44.0 [1.43.0/0.65.0/0.19.0]: https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.43.0 [1.42.0/0.64.0/0.18.0/0.0.16]: https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.42.0 [1.41.0/0.63.0/0.17.0/0.0.15]: https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.41.0 diff --git a/vendor/go.opentelemetry.io/otel/CLAUDE.md b/vendor/go.opentelemetry.io/otel/CLAUDE.md new file mode 100644 index 000000000..dd3c4594f --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/CLAUDE.md @@ -0,0 +1,3 @@ +# Instructions for Claude Code + +@AGENTS.md diff --git a/vendor/go.opentelemetry.io/otel/CONTRIBUTING.md b/vendor/go.opentelemetry.io/otel/CONTRIBUTING.md index 12de3607a..3ec17d683 100644 --- a/vendor/go.opentelemetry.io/otel/CONTRIBUTING.md +++ b/vendor/go.opentelemetry.io/otel/CONTRIBUTING.md @@ -11,6 +11,12 @@ for a summary description of past meetings. To request edit access, join the meeting or get in touch on [Slack](https://cloud-native.slack.com/archives/C01NPAXACKT). +The meeting is open for all to join. We invite everyone to join our +meeting, regardless of your experience level. Whether you're a +seasoned OpenTelemetry developer, just starting your journey, or +simply curious about the work we do, you're more than welcome to +participate! + ## Development You can view and edit the source code by cloning this repository: @@ -746,8 +752,8 @@ Encapsulate setup in constructor functions, ensuring clear ownership and scope: import ( "errors" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) type SDKComponent struct { @@ -808,11 +814,11 @@ func (c *Component) initObservability() { #### Performance -When observability is disabled there should be little to no overhead. +When observability is disabled or the instrument is not `Enabled`, there should be little to no overhead. ```go func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error { - if e.inst != nil { + if e.inst != nil && e.inst.Enabled(ctx) { attrs := expensiveOperation() e.inst.recordSpanInflight(ctx, int64(len(spans)), attrs...) } @@ -829,7 +835,7 @@ func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) } func (i *instrumentation) recordSpanInflight(ctx context.Context, count int64, attrs ...attribute.KeyValue) { - if i == nil || i.inflight == nil { + if i == nil || i.inflight == nil || !i.inflight.Enabled(ctx) { return } i.inflight.Add(ctx, count, metric.WithAttributes(attrs...)) @@ -865,8 +871,12 @@ var ( ) func (i *instrumentation) record(ctx context.Context, value int64, baseAttrs ...attribute.KeyValue) { + if !i.counter.Enabled(ctx) { + return + } attrs := attrPool.Get().(*[]attribute.KeyValue) defer func() { + clear(*attrs) // Clear references to strings/etc to let GC collect them. *attrs = (*attrs)[:0] // Reset. attrPool.Put(attrs) }() @@ -877,6 +887,7 @@ func (i *instrumentation) record(ctx context.Context, value int64, baseAttrs ... addOpt := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*addOpt) *addOpt = (*addOpt)[:0] addOptPool.Put(addOpt) }() @@ -1007,16 +1018,20 @@ Ensure observability measurements receive the correct context, especially for tr ```go func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error { // Use the provided context for observability measurements - e.inst.recordSpanExportStarted(ctx, len(spans)) + if e.inst.Enabled(ctx) { + e.inst.recordSpanExportStarted(ctx, len(spans)) + } err := e.doExport(ctx, spans) - if err != nil { - e.inst.recordSpanExportFailed(ctx, len(spans), err) - } else { - e.inst.recordSpanExportSucceeded(ctx, len(spans)) + if e.inst.Enabled(ctx) { + if err != nil { + e.inst.recordSpanExportFailed(ctx, len(spans), err) + } else { + e.inst.recordSpanExportSucceeded(ctx, len(spans)) + } } - + return err } ``` @@ -1039,7 +1054,7 @@ func (e *Exporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) All observability metrics should follow the [OpenTelemetry Semantic Conventions for SDK metrics](https://github.com/open-telemetry/semantic-conventions/blob/1cf2476ae5e518225a766990a28a6d5602bd5a30/docs/otel/sdk-metrics.md). -Use the metric semantic conventions convenience package [otelconv](./semconv/v1.40.0/otelconv/metric.go). +Use the metric semantic conventions convenience package [otelconv](./semconv/v1.41.0/otelconv/metric.go). ##### Component Identification @@ -1109,6 +1124,68 @@ func TestObservability(t *testing.T) { Test order should not affect results. Ensure that any global state (e.g. component ID counters) is reset between tests. +### Experimental Features + +To support the development of new features in the specification, we use the following patterns to implement in-development features without adding new public artifacts in stable modules. + +#### Experimental behavior with no API artifacts + +Features that change behavior without changing the API (e.g., exemplar collection, auto-generation of identifiers) are implemented behind a feature gate. +The implementation resides in an `/internal/x` package and is activated through environment variables with the `OTEL_GO_X_` prefix (e.g., `OTEL_GO_X_OBSERVABILITY`). +The feature must be documented in a `README.md` file in the `/internal/x` package. + +#### Experimental methods on SDK-only interfaces + +Features that require new methods on SDK interfaces are defined as a new interface in an experimental module (e.g., `go.opentelemetry.io/otel/sdk/x`). +The SDK uses type assertions (without importing the unstable package) to check if passing types implement these experimental interfaces. +The SDK must not depend on the experimental module. + +#### Experimental structs, functions, or interfaces + +Features that don't need any changes to the existing stable package are implemented in an experimental module (e.g., `go.opentelemetry.io/otel/sdk/x`). + +#### Experimental signals and components + +New telemetry signals (e.g., Logs before stabilization) and components (e.g. bridges) are hosted in new, unstable modules (e.g., `go.opentelemetry.io/otel/log` before 1.0.0). +The package should have the final name it will use once stabilized (i.e. not `/x`), and is released at a v0.x.y version to indicate it is not stable. +Most new components are hosted in [opentelemetry-go-contrib](https://github.com/open-telemetry/opentelemetry-go-contrib). + +#### Experimental options for API or SDK functions + +Experimental Options functions are implemented in an experimental module (e.g., `go.opentelemetry.io/otel/sdk/x`). +The return type of the Option function must embed the option's type (e.g. `metric.InstrumentOption`), and have an `Experimental()` method to prevent the API from panicking when the option is used. +The SDK uses type assertions (without importing the unstable package) to check if passing types implement these experimental interfaces. +The SDK must not depend on the experimental module. + +For example: + +```go +type myOption struct { + // Embed the stable option type. + metric.InstrumentOption + value string +} + +// Experimental prevents the API from panicking when the option is used. +func (o myOption) Experimental() {} + +// The SDK can use type assertions to use this function. +func (o myOption) Value() string { return o.value } + +func WithMyOption(value string) metric.InstrumentOption { + return myOption{value: value} +} +``` + +#### Not Supported + +The following kinds of experimental features are **not currently supported** on stable interfaces: + +- Experimental methods on API interfaces +- Experimental fields for API or SDK exported structs + +In some cases forks or long-lived branches may be used for prototyping these features. + ## Approvers and Maintainers ### Maintainers diff --git a/vendor/go.opentelemetry.io/otel/Makefile b/vendor/go.opentelemetry.io/otel/Makefile index 42466f2d6..de63a5e9b 100644 --- a/vendor/go.opentelemetry.io/otel/Makefile +++ b/vendor/go.opentelemetry.io/otel/Makefile @@ -191,8 +191,16 @@ benchmark: $(OTEL_GO_MOD_DIRS:%=benchmark/%) benchmark/%: cd $* && $(GO) test -run='^$$' -bench=. $(ARGS) ./... +# sdk/metric is split into two shards to work around CodSpeed limitations. +# See https://github.com/CodSpeedHQ/codspeed-go/issues/56 +BENCHMARK_SHARDS := $(filter-out ./sdk/metric,$(OTEL_GO_MOD_DIRS)) ./sdk/metric/root ./sdk/metric/internal +benchmark/./sdk/metric/root: + cd ./sdk/metric && $(GO) test -run='^$$' -bench=. $(ARGS) . ./exemplar/... +benchmark/./sdk/metric/internal: + cd ./sdk/metric && $(GO) test -run='^$$' -bench=. $(ARGS) ./internal/... + print-sharded-benchmarks: - @echo $(OTEL_GO_MOD_DIRS) | jq -cR 'split(" ")' + @echo $(BENCHMARK_SHARDS) | jq -cR 'split(" ")' .PHONY: golangci-lint golangci-lint-fix golangci-lint-fix: ARGS=--fix diff --git a/vendor/go.opentelemetry.io/otel/attribute/encoder.go b/vendor/go.opentelemetry.io/otel/attribute/encoder.go index 771dd69c5..ca186d8ac 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/encoder.go +++ b/vendor/go.opentelemetry.io/otel/attribute/encoder.go @@ -105,7 +105,9 @@ func (d *defaultAttrEncoder) Encode(iter Iterator) string { if keyValue.Value.Type() == STRING { copyAndEscape(buf, keyValue.Value.AsString()) } else { - _, _ = buf.WriteString(keyValue.Value.Emit()) + _, _ = buf.WriteString( + keyValue.Value.Emit(), + ) //nolint:staticcheck // Preserve the existing default encoder output. } } return buf.String() diff --git a/vendor/go.opentelemetry.io/otel/attribute/hash.go b/vendor/go.opentelemetry.io/otel/attribute/hash.go index b09caaa6d..92f39ffe7 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/hash.go +++ b/vendor/go.opentelemetry.io/otel/attribute/hash.go @@ -27,6 +27,8 @@ const ( int64SliceID uint64 = 3762322556277578591 // "_[]int64" (little endian) float64SliceID uint64 = 7308324551835016539 // "[]double" (little endian) stringSliceID uint64 = 7453010373645655387 // "[]string" (little endian) + byteSliceID uint64 = 6874028470941080415 // "_[]byte_" (little endian) + sliceID uint64 = 7883494272577650031 // "__slice_" (little endian) emptyID uint64 = 7305809155345288421 // "__empty_" (little endian) ) @@ -42,53 +44,87 @@ func hashKVs(kvs []KeyValue) uint64 { // hashKV returns the xxHash64 hash of kv with h as the base. func hashKV(h xxhash.Hash, kv KeyValue) xxhash.Hash { h = h.String(string(kv.Key)) + return hashValue(h, kv.Value) +} - switch kv.Value.Type() { +func hashValue(h xxhash.Hash, v Value) xxhash.Hash { + switch v.Type() { case BOOL: h = h.Uint64(boolID) - h = h.Uint64(kv.Value.numeric) + h = h.Uint64(v.numeric) case INT64: h = h.Uint64(int64ID) - h = h.Uint64(kv.Value.numeric) + h = h.Uint64(v.numeric) case FLOAT64: h = h.Uint64(float64ID) // Assumes numeric stored with math.Float64bits. - h = h.Uint64(kv.Value.numeric) + h = h.Uint64(v.numeric) case STRING: h = h.Uint64(stringID) - h = h.String(kv.Value.stringly) + h = h.String(v.stringly) case BOOLSLICE: h = h.Uint64(boolSliceID) - rv := reflect.ValueOf(kv.Value.slice) + rv := reflect.ValueOf(v.slice) for i := 0; i < rv.Len(); i++ { h = h.Bool(rv.Index(i).Bool()) } case INT64SLICE: h = h.Uint64(int64SliceID) - rv := reflect.ValueOf(kv.Value.slice) + rv := reflect.ValueOf(v.slice) for i := 0; i < rv.Len(); i++ { h = h.Int64(rv.Index(i).Int()) } case FLOAT64SLICE: h = h.Uint64(float64SliceID) - rv := reflect.ValueOf(kv.Value.slice) + rv := reflect.ValueOf(v.slice) for i := 0; i < rv.Len(); i++ { h = h.Float64(rv.Index(i).Float()) } case STRINGSLICE: h = h.Uint64(stringSliceID) - rv := reflect.ValueOf(kv.Value.slice) + rv := reflect.ValueOf(v.slice) for i := 0; i < rv.Len(); i++ { h = h.String(rv.Index(i).String()) } + case BYTESLICE: + h = h.Uint64(byteSliceID) + h = h.String(v.stringly) + case SLICE: + h = h.Uint64(sliceID) + switch vals := v.slice.(type) { + case [0]Value: + // No values to hash, but the type identifier is still hashed above. + case [1]Value: + h = hashValueSlice(h, vals[:]) + case [2]Value: + h = hashValueSlice(h, vals[:]) + case [3]Value: + h = hashValueSlice(h, vals[:]) + case [4]Value: + h = hashValueSlice(h, vals[:]) + case [5]Value: + h = hashValueSlice(h, vals[:]) + default: + rv := reflect.ValueOf(v.slice) + for i := 0; i < rv.Len(); i++ { + h = hashValue(h, rv.Index(i).Interface().(Value)) + } + } case EMPTY: h = h.Uint64(emptyID) default: // Logging is an alternative, but using the internal logger here // causes an import cycle so it is not done. - v := kv.Value.AsInterface() - msg := fmt.Sprintf("unknown value type: %[1]v (%[1]T)", v) + val := v.AsInterface() + msg := fmt.Sprintf("unknown value type: %[1]v (%[1]T)", val) panic(msg) } return h } + +func hashValueSlice(h xxhash.Hash, vals []Value) xxhash.Hash { + for _, v := range vals { + h = hashValue(h, v) + } + return h +} diff --git a/vendor/go.opentelemetry.io/otel/attribute/key.go b/vendor/go.opentelemetry.io/otel/attribute/key.go index 80a9e5643..cdc7089e8 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/key.go +++ b/vendor/go.opentelemetry.io/otel/attribute/key.go @@ -117,6 +117,28 @@ func (k Key) StringSlice(v []string) KeyValue { } } +// ByteSlice creates a KeyValue instance with a BYTESLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- ByteSlice(name, value). +func (k Key) ByteSlice(v []byte) KeyValue { + return KeyValue{ + Key: k, + Value: ByteSliceValue(v), + } +} + +// Slice creates a KeyValue instance with a SLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Slice(name, values...). +func (k Key) Slice(v ...Value) KeyValue { + return KeyValue{ + Key: k, + Value: SliceValue(v...), + } +} + // Defined reports whether the key is not empty. func (k Key) Defined() bool { return len(k) != 0 diff --git a/vendor/go.opentelemetry.io/otel/attribute/kv.go b/vendor/go.opentelemetry.io/otel/attribute/kv.go index 0cc368018..eeb76a134 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/kv.go +++ b/vendor/go.opentelemetry.io/otel/attribute/kv.go @@ -68,6 +68,16 @@ func StringSlice(k string, v []string) KeyValue { return Key(k).StringSlice(v) } +// ByteSlice creates a KeyValue with a BYTESLICE Value type. +func ByteSlice(k string, v []byte) KeyValue { + return Key(k).ByteSlice(v) +} + +// Slice creates a KeyValue with a SLICE Value type. +func Slice(k string, v ...Value) KeyValue { + return Key(k).Slice(v...) +} + // Stringer creates a new key-value pair with a passed name and a string // value generated by the passed Stringer interface. func Stringer(k string, v fmt.Stringer) KeyValue { diff --git a/vendor/go.opentelemetry.io/otel/attribute/set.go b/vendor/go.opentelemetry.io/otel/attribute/set.go index 6572c98b1..a4b6ce81d 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/set.go +++ b/vendor/go.opentelemetry.io/otel/attribute/set.go @@ -401,7 +401,7 @@ func computeDataFixed(kvs []KeyValue) any { func computeDataReflect(kvs []KeyValue) any { at := reflect.New(reflect.ArrayOf(len(kvs), keyValueType)).Elem() for i, keyValue := range kvs { - *(at.Index(i).Addr().Interface().(*KeyValue)) = keyValue + *at.Index(i).Addr().Interface().(*KeyValue) = keyValue } return at.Interface() } @@ -415,7 +415,7 @@ func (l *Set) MarshalJSON() ([]byte, error) { func (l Set) MarshalLog() any { kvs := make(map[string]string) for _, kv := range l.ToSlice() { - kvs[string(kv.Key)] = kv.Value.Emit() + kvs[string(kv.Key)] = kv.Value.String() } return kvs } diff --git a/vendor/go.opentelemetry.io/otel/attribute/type_string.go b/vendor/go.opentelemetry.io/otel/attribute/type_string.go index 6c04448d6..dbc01d324 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/type_string.go +++ b/vendor/go.opentelemetry.io/otel/attribute/type_string.go @@ -17,11 +17,13 @@ func _() { _ = x[INT64SLICE-6] _ = x[FLOAT64SLICE-7] _ = x[STRINGSLICE-8] + _ = x[BYTESLICE-9] + _ = x[SLICE-10] } -const _Type_name = "EMPTYBOOLINT64FLOAT64STRINGBOOLSLICEINT64SLICEFLOAT64SLICESTRINGSLICE" +const _Type_name = "EMPTYBOOLINT64FLOAT64STRINGBOOLSLICEINT64SLICEFLOAT64SLICESTRINGSLICEBYTESLICESLICE" -var _Type_index = [...]uint8{0, 5, 9, 14, 21, 27, 36, 46, 58, 69} +var _Type_index = [...]uint8{0, 5, 9, 14, 21, 27, 36, 46, 58, 69, 78, 83} func (i Type) String() string { idx := int(i) - 0 diff --git a/vendor/go.opentelemetry.io/otel/attribute/value.go b/vendor/go.opentelemetry.io/otel/attribute/value.go index db04b1326..0529fefae 100644 --- a/vendor/go.opentelemetry.io/otel/attribute/value.go +++ b/vendor/go.opentelemetry.io/otel/attribute/value.go @@ -4,9 +4,14 @@ package attribute // import "go.opentelemetry.io/otel/attribute" import ( + "encoding/base64" "encoding/json" "fmt" + "math" + "reflect" "strconv" + "strings" + "unicode/utf8" attribute "go.opentelemetry.io/otel/attribute/internal" ) @@ -45,6 +50,10 @@ const ( FLOAT64SLICE // STRINGSLICE is a slice of strings Type Value. STRINGSLICE + // BYTESLICE is a slice of bytes Type Value. + BYTESLICE + // SLICE is a slice of Value Type values. + SLICE // INVALID is used for a Value with no value set. // // Deprecated: Use EMPTY instead as an empty value is a valid value. @@ -134,6 +143,19 @@ func StringSliceValue(v []string) Value { return Value{vtype: STRINGSLICE, slice: attribute.SliceValue(v)} } +// ByteSliceValue creates a BYTESLICE Value. +func ByteSliceValue(v []byte) Value { + return Value{ + vtype: BYTESLICE, + stringly: string(v), + } +} + +// SliceValue creates a SLICE Value. +func SliceValue(v ...Value) Value { + return Value{vtype: SLICE, slice: sliceValue(v)} +} + // Type returns a type of the Value. func (v Value) Type() Type { return v.vtype @@ -215,6 +237,59 @@ func (v Value) asStringSlice() []string { return attribute.AsSlice[string](v.slice) } +// AsSlice returns the []Value value. Make sure that the Value's type is +// SLICE. +func (v Value) AsSlice() []Value { + if v.vtype != SLICE { + return nil + } + return v.asSlice() +} + +func (v Value) asSlice() []Value { + switch vals := v.slice.(type) { + case [0]Value: + return []Value{} + case [1]Value: + return []Value{vals[0]} + case [2]Value: + return []Value{vals[0], vals[1]} + case [3]Value: + return []Value{vals[0], vals[1], vals[2]} + case [4]Value: + return []Value{vals[0], vals[1], vals[2], vals[3]} + case [5]Value: + return []Value{vals[0], vals[1], vals[2], vals[3], vals[4]} + default: + return asValueSliceReflect(v.slice) + } +} + +func asValueSliceReflect(v any) []Value { + rv := reflect.ValueOf(v) + if !rv.IsValid() || rv.Kind() != reflect.Array || rv.Type().Elem() != reflect.TypeFor[Value]() { + return nil + } + cpy := make([]Value, rv.Len()) + if len(cpy) > 0 { + _ = reflect.Copy(reflect.ValueOf(cpy), rv) + } + return cpy +} + +// AsByteSlice returns the bytes value. Make sure that the Value's type +// is BYTESLICE. +func (v Value) AsByteSlice() []byte { + if v.vtype != BYTESLICE { + return nil + } + return v.asByteSlice() +} + +func (v Value) asByteSlice() []byte { + return []byte(v.stringly) +} + type unknownValueType struct{} // AsInterface returns Value's data as any. @@ -236,13 +311,60 @@ func (v Value) AsInterface() any { return v.stringly case STRINGSLICE: return v.asStringSlice() + case BYTESLICE: + return v.asByteSlice() + case SLICE: + return v.asSlice() case EMPTY: return nil } return unknownValueType{} } +// String returns a string representation of Value using the +// [OpenTelemetry AnyValue representation for non-OTLP protocols] rules. +// +// Strings are returned as-is without JSON quoting, booleans and integers use +// JSON literals, floating-point values use JSON numbers except that NaN and +// ±Inf are rendered as NaN, Infinity, and -Infinity, byte slices are +// base64-encoded, empty values are the empty string, and slices are encoded as +// JSON arrays. String, byte, and special floating-point values inside arrays +// are encoded as JSON strings, and empty values inside arrays are encoded as +// null. +// +// [OpenTelemetry AnyValue representation for non-OTLP protocols]: https://opentelemetry.io/docs/specs/otel/common/#anyvalue-representation-for-non-otlp-protocols +func (v Value) String() string { + switch v.Type() { + case BOOL: + return strconv.FormatBool(v.AsBool()) + case BOOLSLICE: + return formatBoolSliceValue(v.slice) + case INT64: + return strconv.FormatInt(v.AsInt64(), 10) + case INT64SLICE: + return formatInt64SliceValue(v.slice) + case FLOAT64: + return formatFloat64(v.AsFloat64()) + case FLOAT64SLICE: + return formatFloat64SliceValue(v.slice) + case STRING: + return v.stringly + case STRINGSLICE: + return formatStringSliceValue(v.slice) + case BYTESLICE: + return formatByteSlice(v.stringly) + case SLICE: + return formatValueSliceValue(v.slice) + case EMPTY: + return "" + default: + return "unknown" + } +} + // Emit returns a string representation of Value's data. +// +// Deprecated: Use [Value.String] instead. func (v Value) Emit() string { switch v.Type() { case BOOLSLICE: @@ -273,6 +395,10 @@ func (v Value) Emit() string { return string(j) case STRING: return v.stringly + case BYTESLICE: + return formatByteSlice(v.stringly) + case SLICE: + return formatValueSliceValue(v.slice) case EMPTY: return "" default: @@ -280,6 +406,622 @@ func (v Value) Emit() string { } } +const ( + jsonArrayBracketsLen = len("[]") + boolArrayElemMaxLen = len("false") + int64ArrayElemMaxLen = len("-9223372036854775808") + float64ArrayElemMaxLen = len("-1.7976931348623157e+308") + commaLen = len(",") +) + +func sliceValue(v []Value) any { + switch len(v) { + case 0: + return [0]Value{} + case 1: + return [1]Value{v[0]} + case 2: + return [2]Value{v[0], v[1]} + case 3: + return [3]Value{v[0], v[1], v[2]} + case 4: + return [4]Value{v[0], v[1], v[2], v[3]} + case 5: + return [5]Value{v[0], v[1], v[2], v[3], v[4]} + default: + return sliceValueReflect(v) + } +} + +func sliceValueReflect(v []Value) any { + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[Value]())).Elem() + reflect.Copy(cp, reflect.ValueOf(v)) + return cp.Interface() +} + +func formatBoolSliceValue(v any) string { + switch vals := v.(type) { + case [0]bool: + return "[]" + case [1]bool: + return formatBoolSlice(vals[:]) + case [2]bool: + return formatBoolSlice(vals[:]) + case [3]bool: + return formatBoolSlice(vals[:]) + default: + return formatBoolSliceReflect(v) + } +} + +func formatBoolSlice(vals []bool) string { + var b strings.Builder + appendBoolSlice(&b, vals) + return b.String() +} + +func formatBoolSliceReflect(v any) string { + var b strings.Builder + appendBoolSliceReflect(&b, reflect.ValueOf(v)) + return b.String() +} + +func appendBoolSliceValue(dst *strings.Builder, v any) { + switch vals := v.(type) { + case [0]bool: + _, _ = dst.WriteString("[]") + case [1]bool: + appendBoolSlice(dst, vals[:]) + case [2]bool: + appendBoolSlice(dst, vals[:]) + case [3]bool: + appendBoolSlice(dst, vals[:]) + default: + appendBoolSliceReflect(dst, reflect.ValueOf(v)) + } +} + +func appendBoolSlice(dst *strings.Builder, vals []bool) { + dst.Grow(jsonArrayBracketsLen + len(vals)*(boolArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + for i, val := range vals { + if i > 0 { + _ = dst.WriteByte(',') + } + if val { + _, _ = dst.WriteString("true") + } else { + _, _ = dst.WriteString("false") + } + } + _ = dst.WriteByte(']') +} + +func appendBoolSliceReflect(dst *strings.Builder, rv reflect.Value) { + dst.Grow(jsonArrayBracketsLen + rv.Len()*(boolArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + for i := 0; i < rv.Len(); i++ { + if i > 0 { + _ = dst.WriteByte(',') + } + if rv.Index(i).Bool() { + _, _ = dst.WriteString("true") + } else { + _, _ = dst.WriteString("false") + } + } + _ = dst.WriteByte(']') +} + +func formatInt64SliceValue(v any) string { + switch vals := v.(type) { + case [0]int64: + return "[]" + case [1]int64: + return formatInt64Slice(vals[:]) + case [2]int64: + return formatInt64Slice(vals[:]) + case [3]int64: + return formatInt64Slice(vals[:]) + default: + return formatInt64SliceReflect(v) + } +} + +func formatInt64Slice(vals []int64) string { + var b strings.Builder + appendInt64Slice(&b, vals) + return b.String() +} + +func formatInt64SliceReflect(v any) string { + var b strings.Builder + appendInt64SliceReflect(&b, reflect.ValueOf(v)) + return b.String() +} + +func appendInt64SliceValue(dst *strings.Builder, v any) { + switch vals := v.(type) { + case [0]int64: + _, _ = dst.WriteString("[]") + case [1]int64: + appendInt64Slice(dst, vals[:]) + case [2]int64: + appendInt64Slice(dst, vals[:]) + case [3]int64: + appendInt64Slice(dst, vals[:]) + default: + appendInt64SliceReflect(dst, reflect.ValueOf(v)) + } +} + +func appendInt64Slice(dst *strings.Builder, vals []int64) { + dst.Grow(jsonArrayBracketsLen + len(vals)*(int64ArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + + var buf [int64ArrayElemMaxLen]byte + for i, val := range vals { + if i > 0 { + _ = dst.WriteByte(',') + } + out := strconv.AppendInt(buf[:0], val, 10) + _, _ = dst.Write(out) + } + + _ = dst.WriteByte(']') +} + +func appendInt64SliceReflect(dst *strings.Builder, rv reflect.Value) { + dst.Grow(jsonArrayBracketsLen + rv.Len()*(int64ArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + + var scratch [int64ArrayElemMaxLen]byte + for i := 0; i < rv.Len(); i++ { + if i > 0 { + _ = dst.WriteByte(',') + } + out := strconv.AppendInt(scratch[:0], rv.Index(i).Int(), 10) + _, _ = dst.Write(out) + } + + _ = dst.WriteByte(']') +} + +func formatFloat64(v float64) string { + switch { + case math.IsNaN(v): + return "NaN" + case math.IsInf(v, 1): + return "Infinity" + case math.IsInf(v, -1): + return "-Infinity" + default: + return strconv.FormatFloat(v, 'g', -1, 64) + } +} + +func formatFloat64SliceValue(v any) string { + switch vals := v.(type) { + case [0]float64: + return "[]" + case [1]float64: + return formatFloat64Slice(vals[:]) + case [2]float64: + return formatFloat64Slice(vals[:]) + case [3]float64: + return formatFloat64Slice(vals[:]) + default: + return formatFloat64SliceReflect(v) + } +} + +func formatFloat64Slice(vals []float64) string { + var b strings.Builder + appendFloat64Slice(&b, vals) + return b.String() +} + +func formatFloat64SliceReflect(v any) string { + var b strings.Builder + appendFloat64SliceReflect(&b, reflect.ValueOf(v)) + return b.String() +} + +func appendFloat64SliceValue(dst *strings.Builder, v any) { + switch vals := v.(type) { + case [0]float64: + _, _ = dst.WriteString("[]") + case [1]float64: + appendFloat64Slice(dst, vals[:]) + case [2]float64: + appendFloat64Slice(dst, vals[:]) + case [3]float64: + appendFloat64Slice(dst, vals[:]) + default: + appendFloat64SliceReflect(dst, reflect.ValueOf(v)) + } +} + +func appendFloat64Slice(dst *strings.Builder, vals []float64) { + dst.Grow(jsonArrayBracketsLen + len(vals)*(float64ArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + + var buf [float64ArrayElemMaxLen]byte + for i, val := range vals { + if i > 0 { + _ = dst.WriteByte(',') + } + + switch { + case math.IsNaN(val): + _, _ = dst.WriteString(`"NaN"`) + case math.IsInf(val, 1): + _, _ = dst.WriteString(`"Infinity"`) + case math.IsInf(val, -1): + _, _ = dst.WriteString(`"-Infinity"`) + default: + out := strconv.AppendFloat(buf[:0], val, 'g', -1, 64) + _, _ = dst.Write(out) + } + } + + _ = dst.WriteByte(']') +} + +func appendFloat64SliceReflect(dst *strings.Builder, rv reflect.Value) { + dst.Grow(jsonArrayBracketsLen + rv.Len()*(float64ArrayElemMaxLen+commaLen)) + _ = dst.WriteByte('[') + + var scratch [float64ArrayElemMaxLen]byte + for i := 0; i < rv.Len(); i++ { + if i > 0 { + _ = dst.WriteByte(',') + } + val := rv.Index(i).Float() + switch { + case math.IsNaN(val): + _, _ = dst.WriteString(`"NaN"`) + case math.IsInf(val, 1): + _, _ = dst.WriteString(`"Infinity"`) + case math.IsInf(val, -1): + _, _ = dst.WriteString(`"-Infinity"`) + default: + out := strconv.AppendFloat(scratch[:0], val, 'g', -1, 64) + _, _ = dst.Write(out) + } + } + + _ = dst.WriteByte(']') +} + +func formatStringSliceValue(v any) string { + switch vals := v.(type) { + case [0]string: + return "[]" + case [1]string: + return formatStringSlice(vals[:]) + case [2]string: + return formatStringSlice(vals[:]) + case [3]string: + return formatStringSlice(vals[:]) + default: + return formatStringSliceReflect(v) + } +} + +func formatStringSlice(vals []string) string { + var b strings.Builder + appendStringSlice(&b, vals) + return b.String() +} + +func formatStringSliceReflect(v any) string { + var b strings.Builder + appendStringSliceReflect(&b, reflect.ValueOf(v)) + return b.String() +} + +func appendStringSliceValue(dst *strings.Builder, v any) { + switch vals := v.(type) { + case [0]string: + _, _ = dst.WriteString("[]") + case [1]string: + appendStringSlice(dst, vals[:]) + case [2]string: + appendStringSlice(dst, vals[:]) + case [3]string: + appendStringSlice(dst, vals[:]) + default: + appendStringSliceReflect(dst, reflect.ValueOf(v)) + } +} + +func appendStringSlice(dst *strings.Builder, vals []string) { + size := jsonArrayBracketsLen + for _, val := range vals { + size += len(val) + commaLen + 2 // Account for JSON string quotes and comma. + } + + dst.Grow(size) + _ = dst.WriteByte('[') + for i, val := range vals { + if i > 0 { + _ = dst.WriteByte(',') + } + appendJSONString(dst, val) + } + _ = dst.WriteByte(']') +} + +func appendStringSliceReflect(dst *strings.Builder, rv reflect.Value) { + size := jsonArrayBracketsLen + for i := 0; i < rv.Len(); i++ { + size += len(rv.Index(i).String()) + commaLen + 2 // Account for JSON string quotes and comma. + } + + dst.Grow(size) + _ = dst.WriteByte('[') + for i := 0; i < rv.Len(); i++ { + if i > 0 { + _ = dst.WriteByte(',') + } + appendJSONString(dst, rv.Index(i).String()) + } + _ = dst.WriteByte(']') +} + +func formatByteSlice(v string) string { + var b strings.Builder + appendBase64(&b, v) + return b.String() +} + +func formatValueSliceValue(v any) string { + switch vals := v.(type) { + case [0]Value: + return "[]" + case [1]Value: + return formatValueSlice(vals[:]) + case [2]Value: + return formatValueSlice(vals[:]) + case [3]Value: + return formatValueSlice(vals[:]) + case [4]Value: + return formatValueSlice(vals[:]) + case [5]Value: + return formatValueSlice(vals[:]) + default: + return formatValueSliceReflect(v) + } +} + +func formatValueSlice(vals []Value) string { + var b strings.Builder + appendValueSlice(&b, vals) + return b.String() +} + +func formatValueSliceReflect(v any) string { + var b strings.Builder + appendValueSliceReflect(&b, reflect.ValueOf(v)) + return b.String() +} + +func appendValueSliceValue(dst *strings.Builder, v any) { + switch vals := v.(type) { + case [0]Value: + _, _ = dst.WriteString("[]") + case [1]Value: + appendValueSlice(dst, vals[:]) + case [2]Value: + appendValueSlice(dst, vals[:]) + case [3]Value: + appendValueSlice(dst, vals[:]) + case [4]Value: + appendValueSlice(dst, vals[:]) + case [5]Value: + appendValueSlice(dst, vals[:]) + default: + appendValueSliceReflect(dst, reflect.ValueOf(v)) + } +} + +func appendValueSlice(dst *strings.Builder, vals []Value) { + // Estimate 10 bytes per value for small values and commas. + dst.Grow(jsonArrayBracketsLen + len(vals)*commaLen + len(vals)*10) + _ = dst.WriteByte('[') + for i, val := range vals { + if i > 0 { + _ = dst.WriteByte(',') + } + appendJSONValue(dst, val) + } + _ = dst.WriteByte(']') +} + +func appendValueSliceReflect(dst *strings.Builder, rv reflect.Value) { + // Estimate 10 bytes per value for small values and commas. + dst.Grow(jsonArrayBracketsLen + rv.Len()*commaLen + rv.Len()*10) + _ = dst.WriteByte('[') + for i := 0; i < rv.Len(); i++ { + if i > 0 { + _ = dst.WriteByte(',') + } + appendJSONValue(dst, rv.Index(i).Interface().(Value)) + } + _ = dst.WriteByte(']') +} + +func appendJSONValue(dst *strings.Builder, v Value) { + switch v.Type() { + case BOOL: + if v.AsBool() { + _, _ = dst.WriteString("true") + } else { + _, _ = dst.WriteString("false") + } + case BOOLSLICE: + appendBoolSliceValue(dst, v.slice) + case INT64: + var buf [int64ArrayElemMaxLen]byte + out := strconv.AppendInt(buf[:0], v.AsInt64(), 10) + _, _ = dst.Write(out) + case INT64SLICE: + appendInt64SliceValue(dst, v.slice) + case FLOAT64: + val := v.AsFloat64() + switch { + case math.IsNaN(val): + appendJSONString(dst, "NaN") + case math.IsInf(val, 1): + appendJSONString(dst, "Infinity") + case math.IsInf(val, -1): + appendJSONString(dst, "-Infinity") + default: + var buf [float64ArrayElemMaxLen]byte + out := strconv.AppendFloat(buf[:0], val, 'g', -1, 64) + _, _ = dst.Write(out) + } + case FLOAT64SLICE: + appendFloat64SliceValue(dst, v.slice) + case STRING: + appendJSONString(dst, v.stringly) + case STRINGSLICE: + appendStringSliceValue(dst, v.slice) + case BYTESLICE: + _ = dst.WriteByte('"') + appendBase64(dst, v.stringly) + _ = dst.WriteByte('"') + case SLICE: + appendValueSliceValue(dst, v.slice) + case EMPTY: + _, _ = dst.WriteString("null") + default: + appendJSONString(dst, "unknown") + } +} + +// appendJSONString appends s to dst as a JSON string literal. +// +// This is adapted from the Go standard library's encoding/json +// [appendString implementation]. It keeps the same escaping behavior we need +// here, but writes directly into a strings.Builder and intentionally does not +// apply HTML escaping because the OpenTelemetry non-OTLP AnyValue representation +// only requires JSON array string encoding. We inline this instead of using +// encoding/json so slice formatting avoids allocations and reflection. +// +// [appendString implementation]: https://github.com/golang/go/blob/3b5954c6349d31465dca409b45ab6597e0942d9f/src/encoding/json/encode.go#L998-L1064 +func appendJSONString(dst *strings.Builder, s string) { + const hex = "0123456789abcdef" // For escaping bytes to hex. + + _ = dst.WriteByte('"') + start := 0 + + for i := 0; i < len(s); { + if c := s[i]; c < utf8.RuneSelf { + if c >= 0x20 && c != '\\' && c != '"' { + i++ + continue + } + + if start < i { + _, _ = dst.WriteString(s[start:i]) + } + + switch c { + case '\\', '"': + _ = dst.WriteByte('\\') + _ = dst.WriteByte(c) + case '\b': + _, _ = dst.WriteString(`\b`) + case '\f': + _, _ = dst.WriteString(`\f`) + case '\n': + _, _ = dst.WriteString(`\n`) + case '\r': + _, _ = dst.WriteString(`\r`) + case '\t': + _, _ = dst.WriteString(`\t`) + default: + _, _ = dst.WriteString(`\u00`) + _ = dst.WriteByte(hex[c>>4]) + _ = dst.WriteByte(hex[c&0x0f]) + } + + i++ + start = i + continue + } + + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + if start < i { + _, _ = dst.WriteString(s[start:i]) + } + // Match encoding/json by replacing invalid UTF-8 with U+FFFD. + _, _ = dst.WriteString(`\ufffd`) + i++ + start = i + continue + } + + if r == '\u2028' || r == '\u2029' { + if start < i { + _, _ = dst.WriteString(s[start:i]) + } + // Escape JSONP-sensitive separators unconditionally, like encoding/json. + _, _ = dst.WriteString(`\u202`) + _ = dst.WriteByte(hex[r&0x0f]) + i += size + start = i + continue + } + + i += size + } + + if start < len(s) { + _, _ = dst.WriteString(s[start:]) + } + _ = dst.WriteByte('"') +} + +// This is adapted from the Go standard library's encoding/base64 +// [Encoding.Encode implementation]. It keeps the same encoding behavior we need +// here, but writes directly into a strings.Builder. We inline this instead of using +// encoding/base64 to avoid allocations. +// +// [Encoding.Encode implementation]: https://github.com/golang/go/blob/3b5954c6349d31465dca409b45ab6597e0942d9f/src/encoding/base64/base64.go#L139-L189 +func appendBase64(dst *strings.Builder, s string) { + const encode = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + dst.Grow(base64.StdEncoding.EncodedLen(len(s))) + + i := 0 + for ; i+2 < len(s); i += 3 { + n := uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]) + _ = dst.WriteByte(encode[n>>18&0x3f]) + _ = dst.WriteByte(encode[n>>12&0x3f]) + _ = dst.WriteByte(encode[n>>6&0x3f]) + _ = dst.WriteByte(encode[n&0x3f]) + } + + switch len(s) - i { + case 1: + n := uint32(s[i]) << 16 + _ = dst.WriteByte(encode[n>>18&0x3f]) + _ = dst.WriteByte(encode[n>>12&0x3f]) + _ = dst.WriteByte('=') + _ = dst.WriteByte('=') + case 2: + n := uint32(s[i])<<16 | uint32(s[i+1])<<8 + _ = dst.WriteByte(encode[n>>18&0x3f]) + _ = dst.WriteByte(encode[n>>12&0x3f]) + _ = dst.WriteByte(encode[n>>6&0x3f]) + _ = dst.WriteByte('=') + } +} + // MarshalJSON returns the JSON encoding of the Value. func (v Value) MarshalJSON() ([]byte, error) { var jsonVal struct { diff --git a/vendor/go.opentelemetry.io/otel/baggage/baggage.go b/vendor/go.opentelemetry.io/otel/baggage/baggage.go index 878ffbe43..b290c6d6c 100644 --- a/vendor/go.opentelemetry.io/otel/baggage/baggage.go +++ b/vendor/go.opentelemetry.io/otel/baggage/baggage.go @@ -14,6 +14,10 @@ import ( ) const ( + maxParseErrors = 5 + + // W3C Baggage specification limits. + // https://www.w3.org/TR/baggage/#limits maxMembers = 64 maxBytesPerBaggageString = 8192 @@ -493,9 +497,15 @@ func New(members ...Member) (Baggage, error) { // from the W3C Baggage specification which allows duplicate list-members, but // conforms to the OpenTelemetry Baggage specification. // -// If the baggage-string exceeds the maximum allowed members (64) or bytes -// (8192), members are dropped until the limits are satisfied and an error is -// returned along with the partial result. +// If the raw baggage-string exceeds the maximum allowed bytes (8192), an +// empty Baggage and an error are returned. +// +// Otherwise, members are parsed left-to-right and accumulated until one of +// the following conditions is reached, at which point parsing stops and an +// error is returned alongside the partial result: +// - accepting the next member would cause the encoded baggage to exceed +// 8192 bytes, or +// - the baggage already contains 64 distinct keys. // // Invalid members are skipped and the error is returned along with the // partial result containing the valid members. @@ -504,9 +514,14 @@ func Parse(bStr string) (Baggage, error) { return Baggage{}, nil } + if n := len(bStr); n > maxBytesPerBaggageString { + return Baggage{}, fmt.Errorf("%w: %d", errBaggageBytes, n) + } + b := make(baggage.List) sizes := make(map[string]int) // Track per-key byte sizes var totalBytes int + var parseErrors int var truncateErr error for memberStr := range strings.SplitSeq(bStr, listDelimiter) { // Check member count limit. @@ -517,7 +532,10 @@ func Parse(bStr string) (Baggage, error) { m, err := parseMember(memberStr) if err != nil { - truncateErr = errors.Join(truncateErr, err) + parseErrors++ + if parseErrors <= maxParseErrors { + truncateErr = errors.Join(truncateErr, err) + } continue // skip invalid member, keep processing } @@ -553,6 +571,10 @@ func Parse(bStr string) (Baggage, error) { totalBytes = newTotalBytes } + if dropped := parseErrors - maxParseErrors; dropped > 0 { + truncateErr = errors.Join(truncateErr, fmt.Errorf("and %d more invalid member(s)", dropped)) + } + if len(b) == 0 { return Baggage{}, truncateErr } diff --git a/vendor/go.opentelemetry.io/otel/dependencies.Dockerfile b/vendor/go.opentelemetry.io/otel/dependencies.Dockerfile index 7a9b3c055..74fa510bc 100644 --- a/vendor/go.opentelemetry.io/otel/dependencies.Dockerfile +++ b/vendor/go.opentelemetry.io/otel/dependencies.Dockerfile @@ -1,4 +1,4 @@ # This is a renovate-friendly source of Docker images. FROM python:3.13.6-slim-bullseye@sha256:e98b521460ee75bca92175c16247bdf7275637a8faaeb2bcfa19d879ae5c4b9a AS python -FROM otel/weaver:v0.22.1@sha256:33ae522ae4b71c1c562563c1d81f46aa0f79f088a0873199143a1f11ac30e5c9 AS weaver +FROM otel/weaver:v0.23.0@sha256:7984ecb55b859eb3034ae9d836c4eeda137e2bdd0873b7ba2bb6c3d24d6ff457 AS weaver FROM avtodev/markdown-lint:v1@sha256:6aeedc2f49138ce7a1cd0adffc1b1c0321b841dc2102408967d9301c031949ee AS markdown diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/client.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/client.go index 7b7f90729..22b845053 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/client.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/client.go @@ -6,6 +6,7 @@ package otlpmetricgrpc // import "go.opentelemetry.io/otel/exporters/otlp/otlpme import ( "context" "errors" + "fmt" "time" colmetricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" @@ -15,6 +16,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf" @@ -22,9 +24,10 @@ import ( ) type client struct { - metadata metadata.MD - exportTimeout time.Duration - requestFunc retry.RequestFunc + metadata metadata.MD + exportTimeout time.Duration + maxRequestSize int + requestFunc retry.RequestFunc // ourConn keeps track of where conn was created: true if created here in // NewClient, or false if passed with an option. This is important on @@ -38,9 +41,10 @@ type client struct { // newClient creates a new gRPC metric client. func newClient(_ context.Context, cfg oconf.Config) (*client, error) { c := &client{ - exportTimeout: cfg.Metrics.Timeout, - requestFunc: cfg.RetryConfig.RequestFunc(retryable), - conn: cfg.GRPCConn, + exportTimeout: cfg.Metrics.Timeout, + maxRequestSize: cfg.Metrics.MaxRequestSize, + requestFunc: cfg.RetryConfig.RequestFunc(retryable), + conn: cfg.GRPCConn, } if len(cfg.Metrics.Headers) > 0 { @@ -115,10 +119,15 @@ func (c *client) UploadMetrics(ctx context.Context, protoMetrics *metricpb.Resou ctx, cancel := c.exportContext(ctx) defer cancel() + pbRequest := &colmetricpb.ExportMetricsServiceRequest{ + ResourceMetrics: []*metricpb.ResourceMetrics{protoMetrics}, + } + if maxSize := c.maxRequestSize; maxSize > 0 && proto.Size(pbRequest) > maxSize { + return fmt.Errorf("request message too large: exceeded %d bytes", maxSize) + } + return errors.Join(uploadErr, c.requestFunc(ctx, func(iCtx context.Context) error { - resp, err := c.msc.Export(iCtx, &colmetricpb.ExportMetricsServiceRequest{ - ResourceMetrics: []*metricpb.ResourceMetrics{protoMetrics}, - }) + resp, err := c.msc.Export(iCtx, pbRequest) if resp != nil && resp.PartialSuccess != nil { msg := resp.PartialSuccess.GetErrorMessage() n := resp.PartialSuccess.GetRejectedDataPoints() diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/config.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/config.go index c831bb60b..47eb85e9f 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/config.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/config.go @@ -231,6 +231,16 @@ func WithTimeout(duration time.Duration) Option { return wrappedOption{oconf.WithTimeout(duration)} } +// WithMaxRequestSize sets the maximum size, in bytes, of a serialized export +// request, before compression, that the exporter will send. +// +// If size is less than or equal to zero, no request-size limit is applied. +// Disabling the limit is not recommended because it can lead to excessive +// resource consumption or abuse. +func WithMaxRequestSize(size int) Option { + return wrappedOption{oconf.WithMaxRequestSize(size)} +} + // WithRetry sets the retry policy for transient retryable errors that are // returned by the target endpoint. // diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/doc.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/doc.go index dcd8de5df..62b05626f 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/doc.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/doc.go @@ -77,6 +77,9 @@ default aggregation to use for histogram instruments. Supported values: The configuration can be overridden by [WithAggregationSelector] option. +See [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x] for information about +the experimental features. + [W3C Baggage HTTP Header Content Format]: https://www.w3.org/TR/baggage/#header-content [Explicit Bucket Histogram Aggregation]: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.26.0/specification/metrics/sdk.md#explicit-bucket-histogram-aggregation [Base2 Exponential Bucket Histogram Aggregation]: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.26.0/specification/metrics/sdk.md#base2-exponential-bucket-histogram-aggregation diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go index 35cdf4661..b567800e3 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go @@ -11,8 +11,11 @@ import ( metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/transform" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -31,6 +34,9 @@ type Exporter struct { aggregationSelector metric.AggregationSelector shutdownOnce sync.Once + + // Self-observability metrics + inst *observ.Instrumentation } func newExporter(c *client, cfg oconf.Config) (*Exporter, error) { @@ -46,12 +52,27 @@ func newExporter(c *client, cfg oconf.Config) (*Exporter, error) { as = metric.DefaultAggregationSelector } + var inst *observ.Instrumentation + var initErr error + if x.Observability.Enabled() { + var err error + inst, err = observ.NewInstrumentation( + counter.NextExporterID(), + c.conn.CanonicalTarget(), + ) + if err != nil { + initErr = err + } + } + return &Exporter{ client: c, temporalitySelector: ts, aggregationSelector: as, - }, nil + + inst: inst, + }, initErr } // Temporality returns the Temporality to use for an instrument kind. @@ -72,10 +93,18 @@ func (e *Exporter) Export(ctx context.Context, rm *metricdata.ResourceMetrics) e defer global.Debug("OTLP/gRPC exporter export", "Data", rm) otlpRm, err := transform.ResourceMetrics(rm) + + // Track export operation for self-observability + op := e.inst.TrackExport(ctx, otlpRm) + + var upErr error + defer func() { op.End(upErr) }() + // Best effort upload of transformable metrics. e.clientMu.Lock() - upErr := e.client.UploadMetrics(ctx, otlpRm) + upErr = e.client.UploadMetrics(ctx, otlpRm) e.clientMu.Unlock() + if upErr != nil { if err == nil { return fmt.Errorf("failed to upload metrics: %w", upErr) diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter/counter.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter/counter.go new file mode 100644 index 000000000..df6b830d3 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter/counter.go @@ -0,0 +1,31 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/counter/counter.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package counter provides a simple counter for generating unique IDs. +// +// This package is used to generate unique IDs while allowing testing packages +// to reset the counter. +package counter // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter" + +import "sync/atomic" + +// exporterN is a global 0-based count of the number of exporters created. +var exporterN atomic.Int64 + +// NextExporterID returns the next unique ID for an exporter. +func NextExporterID() int64 { + const inc = 1 + return exporterN.Add(inc) - inc +} + +// SetExporterID sets the exporter ID counter to v and returns the previous +// value. +// +// This function is useful for testing purposes, allowing you to reset the +// counter. It should not be used in production code. +func SetExporterID(v int64) int64 { + return exporterN.Swap(v) +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/gen.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/gen.go index f6e4f0d07..c94f85ef4 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/gen.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/gen.go @@ -7,6 +7,9 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/o //go:generate gotmpl --body=../../../../../internal/shared/otlp/partialsuccess.go.tmpl "--data={}" --out=partialsuccess.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/partialsuccess_test.go.tmpl "--data={}" --out=partialsuccess_test.go +//go:generate gotmpl --body=../../../../../internal/shared/otlp/observ/target.go.tmpl "--data={ \"pkg\": \"observ\", \"pkg_path\": \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ\" }" --out=observ/target.go +//go:generate gotmpl --body=../../../../../internal/shared/otlp/observ/target_test.go.tmpl "--data={ \"pkg\": \"observ\" }" --out=observ/target_test.go + //go:generate gotmpl --body=../../../../../internal/shared/otlp/retry/retry.go.tmpl "--data={}" --out=retry/retry.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/retry/retry_test.go.tmpl "--data={}" --out=retry/retry_test.go @@ -30,3 +33,9 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/o //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/error_test.go.tmpl "--data={}" --out=transform/error_test.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/metricdata.go.tmpl "--data={}" --out=transform/metricdata.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/metricdata_test.go.tmpl "--data={}" --out=transform/metricdata_test.go + +//go:generate gotmpl --body=../../../../../internal/shared/counter/counter.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/counter\" }" --out=counter/counter.go +//go:generate gotmpl --body=../../../../../internal/shared/counter/counter_test.go.tmpl "--data={}" --out=counter/counter_test.go + +//go:generate gotmpl --body=../../../../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc\" }" --out=x/x.go +//go:generate gotmpl --body=../../../../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/instrumentation.go new file mode 100644 index 000000000..2b4dbb5d8 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/instrumentation.go @@ -0,0 +1,343 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package observ provides self-observability metrics for OTLP metric exporters. +// This is an experimental feature controlled by the x.Observability feature flag. +package observ // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ" + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/internal/global" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x" +) + +var ( + attrPool = sync.Pool{ + New: func() any { + // Pre-allocate for common attributes and dynamic error attributes. + const n = 1 /* otel.component.type */ + + 1 /* otel.component.name */ + + 1 /* server.address */ + + 1 /* server.port */ + + 1 /* error.type */ + + 1 /* rpc.grpc.status_code */ + s := make([]attribute.KeyValue, 0, n) + return &s + }, + } + + recOptPool = sync.Pool{ + New: func() any { + o := make([]metric.RecordOption, 0, 1) + return &o + }, + } +) + +// Instrumentation holds the self-observability metric instruments for an OTLP metric exporter. +type Instrumentation struct { + exported otelconv.SDKExporterMetricDataPointExported + inflight otelconv.SDKExporterMetricDataPointInflight + duration otelconv.SDKExporterOperationDuration + attrs []attribute.KeyValue + addOpt metric.AddOption + recOpt metric.RecordOption +} + +// NewInstrumentation returns instrumentation for an OTLP over gRPC metric +// exporter with the provided ID using the global MeterProvider. +// +// The id should be the unique exporter instance ID. It is used +// to set the "component.name" attribute. +// +// The target is the endpoint the exporter is exporting to. +// +// If the experimental observability is disabled, nil is returned. +func NewInstrumentation(id int64, target string) (*Instrumentation, error) { + if !x.Observability.Enabled() { + return nil, nil + } + + em := &Instrumentation{} + + meter := otel.GetMeterProvider().Meter( + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc", + metric.WithInstrumentationVersion(sdk.Version()), + metric.WithSchemaURL(semconv.SchemaURL), + ) + + var err error + var instrumentErr error + + em.exported, instrumentErr = otelconv.NewSDKExporterMetricDataPointExported(meter) + if instrumentErr != nil { + err = errors.Join(err, fmt.Errorf("failed to create exported metric: %w", instrumentErr)) + } + + em.inflight, instrumentErr = otelconv.NewSDKExporterMetricDataPointInflight(meter) + if instrumentErr != nil { + err = errors.Join(err, fmt.Errorf("failed to create inflight metric: %w", instrumentErr)) + } + + em.duration, instrumentErr = otelconv.NewSDKExporterOperationDuration(meter) + if instrumentErr != nil { + err = errors.Join(err, fmt.Errorf("failed to create duration metric: %w", instrumentErr)) + } + + em.attrs = BaseAttrs(id, target) + + attrSet := attribute.NewSet(em.attrs...) + em.addOpt = metric.WithAttributeSet(attrSet) + em.recOpt = metric.WithAttributeSet(attribute.NewSet(append( + []attribute.KeyValue{semconv.RPCResponseStatusCode(codes.OK.String())}, + em.attrs..., + )...)) + + return em, err +} + +// ComponentName returns the component name for the exporter with the +// provided ID. +func ComponentName(id int64) string { + t := string(otelconv.ComponentTypeOtlpGRPCMetricExporter) + return fmt.Sprintf("%s/%d", t, id) +} + +// BaseAttrs returns the base attributes for the exporter with the provided ID +// and target. +// +// The id should be the unique exporter instance ID. It is used +// to set the "component.name" attribute. +// +// The target is the gRPC target the exporter is exporting to. It is expected +// to be the output of the Client's CanonicalTarget method. +func BaseAttrs(id int64, target string) []attribute.KeyValue { + host, port, err := ParseCanonicalTarget(target) + if err != nil || (host == "" && port < 0) { + if err != nil { + global.Debug("failed to parse target", "target", target, "error", err) + } + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeKey.String(string(otelconv.ComponentTypeOtlpGRPCMetricExporter)), + } + } + + // Do not use append so the slice is exactly allocated. + + if port < 0 { + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeKey.String(string(otelconv.ComponentTypeOtlpGRPCMetricExporter)), + semconv.ServerAddress(host), + } + } + + if host == "" { + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeKey.String(string(otelconv.ComponentTypeOtlpGRPCMetricExporter)), + semconv.ServerPort(port), + } + } + + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeKey.String(string(otelconv.ComponentTypeOtlpGRPCMetricExporter)), + semconv.ServerAddress(host), + semconv.ServerPort(port), + } +} + +// TrackExport tracks an export operation and returns an ExportOp to complete the tracking. +func (em *Instrumentation) TrackExport(ctx context.Context, rm *metricpb.ResourceMetrics) ExportOp { + if em == nil { + return ExportOp{} + } + start := time.Now() + + var dataPointCount int64 + inflightEnabled := em.inflight.Enabled(ctx) + exportedEnabled := em.exported.Enabled(ctx) + + if inflightEnabled || exportedEnabled { + dataPointCount = countProtoDataPoints(rm) + } + + if inflightEnabled { + em.inflight.Inst().Add(ctx, dataPointCount, em.addOpt) + } + + return ExportOp{ + ctx: ctx, + start: start, + dataPointCount: dataPointCount, + inst: em, + } +} + +// ExportOp tracks the operation being observed by [Instrumentation.TrackExport]. +type ExportOp struct { + ctx context.Context + start time.Time + dataPointCount int64 + + inst *Instrumentation +} + +// End completes the observation of the operation being observed by a call to +// [Instrumentation.TrackExport]. +// +// Any error that is encountered is provided as err. +func (e ExportOp) End(err error) { + if e.inst == nil { + return + } + if e.inst.inflight.Enabled(e.ctx) { + e.inst.inflight.Inst().Add(e.ctx, -e.dataPointCount, e.inst.addOpt) + } + + success := successful(e.dataPointCount, err) + // Record successfully exported data points, even if the value is 0 which are + // meaningful to distribution aggregations. + if e.inst.exported.Enabled(e.ctx) { + e.inst.exported.Inst().Add(e.ctx, success, e.inst.addOpt) + } + + if err != nil && e.inst.exported.Enabled(e.ctx) { + attrsPtr := attrPool.Get().(*[]attribute.KeyValue) + defer func() { + *attrsPtr = (*attrsPtr)[:0] + attrPool.Put(attrsPtr) + }() + + *attrsPtr = append(*attrsPtr, e.inst.attrs...) + *attrsPtr = append(*attrsPtr, semconv.ErrorType(err)) + + set := attribute.NewSet(*attrsPtr...) + e.inst.exported.Inst().Add(e.ctx, e.dataPointCount-success, metric.WithAttributeSet(set)) + } + + if e.inst.duration.Enabled(e.ctx) { + d := time.Since(e.start).Seconds() + if err != nil { + recOptPtr := recOptPool.Get().(*[]metric.RecordOption) + defer func() { + *recOptPtr = (*recOptPtr)[:0] + recOptPool.Put(recOptPtr) + }() + + attrsPtr := attrPool.Get().(*[]attribute.KeyValue) + defer func() { + *attrsPtr = (*attrsPtr)[:0] + attrPool.Put(attrsPtr) + }() + + *attrsPtr = append(*attrsPtr, e.inst.attrs...) + *attrsPtr = append( + *attrsPtr, + semconv.ErrorType(err), + semconv.RPCResponseStatusCode(status.Code(err).String()), + ) + + set := attribute.NewSet(*attrsPtr...) + *recOptPtr = append(*recOptPtr, metric.WithAttributeSet(set)) + + e.inst.duration.Inst().Record(e.ctx, d, *recOptPtr...) + } else { + e.inst.duration.Inst().Record(e.ctx, d, e.inst.recOpt) + } + } +} + +// countProtoDataPoints counts the total number of data points in a ResourceMetrics. +func countProtoDataPoints(rm *metricpb.ResourceMetrics) int64 { + if rm == nil { + return 0 + } + + var total int64 + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch data := m.Data.(type) { + case *metricpb.Metric_Gauge: + if data.Gauge != nil { + total += int64(len(data.Gauge.DataPoints)) + } + case *metricpb.Metric_Sum: + if data.Sum != nil { + total += int64(len(data.Sum.DataPoints)) + } + case *metricpb.Metric_Histogram: + if data.Histogram != nil { + total += int64(len(data.Histogram.DataPoints)) + } + case *metricpb.Metric_ExponentialHistogram: + if data.ExponentialHistogram != nil { + total += int64(len(data.ExponentialHistogram.DataPoints)) + } + case *metricpb.Metric_Summary: + if data.Summary != nil { + total += int64(len(data.Summary.DataPoints)) + } + } + } + } + return total +} + +// successful returns the number of successfully exported data points out of the n +// that were exported based on the provided error. +// +// If err is nil, n is returned. All data points were successfully exported. +// +// If err is not nil and not an [internal.PartialSuccess] error, 0 is returned. +// It is assumed all data points failed to be exported. +// +// If err is an [internal.PartialSuccess] error, the number of successfully +// exported data points is computed by subtracting the RejectedItems field from n. If +// RejectedItems is negative, n is returned. If RejectedItems is greater than +// n, 0 is returned. +func successful(n int64, err error) int64 { + if err == nil { + return n // All data points successfully exported. + } + // Split rejection calculation so successful is inlinable. + return n - rejected(n, err) +} + +var errPartialPool = &sync.Pool{ + New: func() any { return new(internal.PartialSuccess) }, +} + +// rejected returns how many out of the n data points exporter were rejected based on +// the provided non-nil err. +func rejected(n int64, err error) int64 { + ps := errPartialPool.Get().(*internal.PartialSuccess) + defer errPartialPool.Put(ps) + // Check for partial success. + if errors.As(err, ps) { + // Bound RejectedItems to [0, n]. This should not be needed, + // but be defensive as this is from an external source. + return min(max(ps.RejectedItems, 0), n) + } + return n // All data points rejected. +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/target.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/target.go new file mode 100644 index 000000000..0be4501ec --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ/target.go @@ -0,0 +1,143 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/otlp/observ/target.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package observ // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/observ" + +import ( + "errors" + "fmt" + "net" + "net/netip" + "strconv" + "strings" +) + +const ( + schemeUnix = "unix" + schemeUnixAbstract = "unix-abstract" +) + +// ParseCanonicalTarget parses a target string and returns the extracted host +// (domain address or IP), the target port, or an error. +// +// If no port is specified, -1 is returned. +// +// If no host is specified, an empty string is returned. +// +// The target string is expected to always have the form +// "://[authority]/". For example: +// - "dns:///example.com:42" +// - "dns://8.8.8.8/example.com:42" +// - "unix:///path/to/socket" +// - "unix-abstract:///socket-name" +// - "passthrough:///192.34.2.1:42" +// +// The target is expected to come from the CanonicalTarget method of a gRPC +// Client. +func ParseCanonicalTarget(target string) (string, int, error) { + const sep = "://" + + // Find scheme. Do not allocate the string by using url.Parse. + idx := strings.Index(target, sep) + if idx == -1 { + return "", -1, fmt.Errorf("invalid target %q: missing scheme", target) + } + scheme, endpoint := target[:idx], target[idx+len(sep):] + + // Check for unix schemes. + if scheme == schemeUnix || scheme == schemeUnixAbstract { + return parseUnix(endpoint) + } + + // Strip leading slash and any authority. + if i := strings.Index(endpoint, "/"); i != -1 { + endpoint = endpoint[i+1:] + } + + // DNS, passthrough, and custom resolvers. + return parseEndpoint(endpoint) +} + +// parseUnix parses unix socket targets. +func parseUnix(endpoint string) (string, int, error) { + // Format: unix[-abstract]://path + // + // We should have "/path" (empty authority) if valid. + if len(endpoint) >= 1 && endpoint[0] == '/' { + // Return the full path including leading slash. + return endpoint, -1, nil + } + + // If there's no leading slash, it means there might be an authority + // Check for authority case (should error): "authority/path" + if slashIdx := strings.Index(endpoint, "/"); slashIdx > 0 { + return "", -1, fmt.Errorf("invalid (non-empty) authority: %s", endpoint[:slashIdx]) + } + + return "", -1, errors.New("invalid unix target format") +} + +// parseEndpoint parses an endpoint from a gRPC target. +// +// It supports the following formats: +// - "host" +// - "host%zone" +// - "host:port" +// - "host%zone:port" +// - "ipv4" +// - "ipv4%zone" +// - "ipv4:port" +// - "ipv4%zone:port" +// - "ipv6" +// - "ipv6%zone" +// - "[ipv6]" +// - "[ipv6%zone]" +// - "[ipv6]:port" +// - "[ipv6%zone]:port" +// +// It returns the host or host%zone (domain address or IP), the port (or -1 if +// not specified), or an error if the input is not a valid. +func parseEndpoint(endpoint string) (string, int, error) { + // First check if the endpoint is just an IP address. + if ip := parseIP(endpoint); ip != "" { + return ip, -1, nil + } + + // If there's no colon, there is no port (IPv6 with no port checked above). + if !strings.Contains(endpoint, ":") { + return endpoint, -1, nil + } + + host, portStr, err := net.SplitHostPort(endpoint) + if err != nil { + return "", -1, fmt.Errorf("invalid host:port %q: %w", endpoint, err) + } + + const base, bitSize = 10, 16 + port16, err := strconv.ParseUint(portStr, base, bitSize) + if err != nil { + return "", -1, fmt.Errorf("invalid port %q: %w", portStr, err) + } + port := int(port16) // port is guaranteed to be in the range [0, 65535]. + + return host, port, nil +} + +// parseIP attempts to parse the entire endpoint as an IP address. +// It returns the normalized string form of the IP if successful, +// or an empty string if parsing fails. +func parseIP(ip string) string { + // Strip leading and trailing brackets for IPv6 addresses. + if len(ip) >= 2 && ip[0] == '[' && ip[len(ip)-1] == ']' { + ip = ip[1 : len(ip)-1] + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return "" + } + // Return the normalized string form of the IP. + return addr.String() +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf/options.go index 758d1ea32..b89d7eeb7 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/oconf/options.go @@ -35,6 +35,9 @@ const ( // DefaultMetricsPath is a default URL path for endpoint that // receives metrics. DefaultMetricsPath string = "/v1/metrics" + // DefaultMaxRequestSize is the default maximum size of a serialized export + // request, before compression. + DefaultMaxRequestSize int = 64 * 1024 * 1024 // DefaultBackoff is a default base backoff time used in the // exponential backoff strategy. DefaultBackoff time.Duration = 300 * time.Millisecond @@ -49,13 +52,14 @@ type ( HTTPTransportProxyFunc func(*http.Request) (*url.URL, error) SignalConfig struct { - Endpoint string - Insecure bool - TLSCfg *tls.Config - Headers map[string]string - Compression Compression - Timeout time.Duration - URLPath string + Endpoint string + Insecure bool + TLSCfg *tls.Config + Headers map[string]string + Compression Compression + MaxRequestSize int + Timeout time.Duration + URLPath string TemporalitySelector metric.TemporalitySelector AggregationSelector metric.AggregationSelector @@ -87,10 +91,11 @@ type ( func NewHTTPConfig(opts ...HTTPOption) Config { cfg := Config{ Metrics: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), - URLPath: DefaultMetricsPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), + URLPath: DefaultMetricsPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, TemporalitySelector: metric.DefaultTemporalitySelector, AggregationSelector: metric.DefaultAggregationSelector, @@ -123,10 +128,11 @@ func cleanPath(urlPath string, defaultPath string) string { func NewGRPCConfig(opts ...GRPCOption) Config { cfg := Config{ Metrics: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), - URLPath: DefaultMetricsPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), + URLPath: DefaultMetricsPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, TemporalitySelector: metric.DefaultTemporalitySelector, AggregationSelector: metric.DefaultAggregationSelector, @@ -354,6 +360,13 @@ func WithTimeout(duration time.Duration) GenericOption { }) } +func WithMaxRequestSize(size int) GenericOption { + return newGenericOption(func(cfg Config) Config { + cfg.Metrics.MaxRequestSize = size + return cfg + }) +} + func WithTemporalitySelector(selector metric.TemporalitySelector) GenericOption { return newGenericOption(func(cfg Config) Config { cfg.Metrics.TemporalitySelector = selector diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/transform/attribute.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/transform/attribute.go index 3e616a927..96420d421 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/transform/attribute.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/transform/attribute.go @@ -81,6 +81,16 @@ func Value(v attribute.Value) *cpb.AnyValue { av.Value = &cpb.AnyValue_StringValue{ StringValue: v.AsString(), } + case attribute.BYTESLICE: + av.Value = &cpb.AnyValue_BytesValue{ + BytesValue: v.AsByteSlice(), + } + case attribute.SLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: attrValues(v.AsSlice()), + }, + } case attribute.STRINGSLICE: av.Value = &cpb.AnyValue_ArrayValue{ ArrayValue: &cpb.ArrayValue{ @@ -143,3 +153,11 @@ func stringSliceValues(vals []string) []*cpb.AnyValue { } return converted } + +func attrValues(vals []attribute.Value) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = Value(v) + } + return converted +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/README.md b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/README.md new file mode 100644 index 000000000..8da02ee13 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/README.md @@ -0,0 +1,55 @@ +# Experimental Features + +The OTLP gRPC metric exporter contains features that have not yet stabilized in the OpenTelemetry specification. +These features are added to the OpenTelemetry Go OTLP exporters prior to stabilization in the specification so that users can start experimenting with them and provide feedback. + +These features may change in backwards incompatible ways as feedback is applied. +See the [Compatibility and Stability](#compatibility-and-stability) section for more information. + +## Features + +- [Self-Observability](#self-observability) + +### Self-Observability + +The OTLP gRPC metric exporter can emit self-observability metrics to track its own operation. + +This experimental feature can be enabled by setting the `OTEL_GO_X_OBSERVABILITY` environment variable. +The value must be the case-insensitive string of `"true"` to enable the feature. +All other values are ignored. + +When enabled, the exporter will emit the following metrics using the global MeterProvider: + +- `otel.sdk.exporter.metric_data_point.exported`: Counter tracking successfully exported data points +- `otel.sdk.exporter.metric_data_point.inflight`: UpDownCounter tracking data points currently being exported +- `otel.sdk.exporter.operation.duration`: Histogram tracking export operation duration in seconds + +All metrics include attributes identifying the exporter component and destination server: + +- `otel.component.type`: Type of component (e.g., "otlp_grpc_metric_exporter") +- `otel.component.name`: Unique component instance name (e.g., "otlp_grpc_metric_exporter/0") +- `server.address`: Server hostname or address +- `server.port`: Server port number + +#### Examples + +Enable self-observability metrics. + +```console +export OTEL_GO_X_OBSERVABILITY=true +``` + +Disable self-observability metrics. + +```console +unset OTEL_GO_X_OBSERVABILITY +``` + +## Compatibility and Stability + +Experimental features do not fall within the scope of the OpenTelemetry Go versioning and stability [policy](../../../../../../VERSIONING.md). +These features may be removed or modified in successive version releases, including patch versions. + +When an experimental feature is promoted to a stable feature, a migration path will be included in the changelog entry of the release. +There is no guarantee that any environment variable feature flags that enabled the experimental feature will be supported by the stable version. +If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/observ.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/observ.go new file mode 100644 index 000000000..cc055ea47 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/observ.go @@ -0,0 +1,22 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package x // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x" + +import "strings" + +// Observability is an experimental feature flag that defines if OTLP +// gRPC metric exporter should include self-observability metrics. +// +// To enable this feature set the OTEL_GO_X_OBSERVABILITY environment variable +// to the case-insensitive string value of "true" (i.e. "True" and "TRUE" +// will also enable this). +var Observability = newFeature( + []string{"OBSERVABILITY"}, + func(v string) (string, bool) { + if strings.EqualFold(v, "true") { + return v, true + } + return "", false + }, +) diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/x.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/x.go new file mode 100644 index 000000000..d42827877 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x/x.go @@ -0,0 +1,58 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/x/x.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package x documents experimental features for [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc]. +package x // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/x" + +import ( + "os" +) + +// Feature is an experimental feature control flag. It provides a uniform way +// to interact with these feature flags and parse their values. +type Feature[T any] struct { + keys []string + parse func(v string) (T, bool) +} + +func newFeature[T any](suffix []string, parse func(string) (T, bool)) Feature[T] { + const envKeyRoot = "OTEL_GO_X_" + keys := make([]string, 0, len(suffix)) + for _, s := range suffix { + keys = append(keys, envKeyRoot+s) + } + return Feature[T]{ + keys: keys, + parse: parse, + } +} + +// Keys returns the environment variable keys that can be set to enable the +// feature. +func (f Feature[T]) Keys() []string { return f.keys } + +// Lookup returns the user configured value for the feature and true if the +// user has enabled the feature. Otherwise, if the feature is not enabled, a +// zero-value and false are returned. +func (f Feature[T]) Lookup() (v T, ok bool) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/62effed618589a0bec416a87e559c0a9d96289bb/specification/configuration/sdk-environment-variables.md#parsing-empty-value + // + // > The SDK MUST interpret an empty value of an environment variable the + // > same way as when the variable is unset. + for _, key := range f.keys { + vRaw := os.Getenv(key) + if vRaw != "" { + return f.parse(vRaw) + } + } + return v, ok +} + +// Enabled reports whether the feature is enabled. +func (f Feature[T]) Enabled() bool { + _, ok := f.Lookup() + return ok +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/version.go index 0606277b3..8ee005c63 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc/version.go @@ -5,5 +5,5 @@ package otlpmetricgrpc // import "go.opentelemetry.io/otel/exporters/otlp/otlpme // Version is the current release version of the OpenTelemetry OTLP over gRPC metrics exporter in use. func Version() string { - return "1.43.0" + return "1.44.0" } diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/client.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/client.go index 1958f9d1b..b7ca17fe3 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/client.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/client.go @@ -23,16 +23,21 @@ import ( "google.golang.org/protobuf/proto" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/oconf" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/retry" ) type client struct { // req is cloned for every upload the client makes. - req *http.Request - compression Compression - requestFunc retry.RequestFunc - httpClient *http.Client + req *http.Request + compression Compression + maxRequestSize int + requestFunc retry.RequestFunc + httpClient *http.Client + + inst *observ.Instrumentation } // Keep it in sync with golang's DefaultTransport from net/http! We @@ -111,12 +116,17 @@ func newClient(cfg oconf.Config) (*client, error) { } req.Header.Set("Content-Type", "application/x-protobuf") + // Initialize the instrumentation. + inst, err := observ.NewInstrumentation(counter.NextExporterID(), cfg.Metrics.Endpoint) + return &client{ - compression: Compression(cfg.Metrics.Compression), - req: req, - requestFunc: cfg.RetryConfig.RequestFunc(evaluate), - httpClient: httpClient, - }, nil + compression: Compression(cfg.Metrics.Compression), + maxRequestSize: cfg.Metrics.MaxRequestSize, + req: req, + requestFunc: cfg.RetryConfig.RequestFunc(evaluate), + httpClient: httpClient, + inst: inst, + }, err } // Shutdown shuts down the client, freeing all resources. @@ -146,11 +156,20 @@ func (c *client) UploadMetrics(ctx context.Context, protoMetrics *metricpb.Resou if err != nil { return err } + if maxSize := c.maxRequestSize; maxSize > 0 && len(body) > maxSize { + return fmt.Errorf("request body too large: exceeded %d bytes", maxSize) + } request, err := c.newRequest(ctx, body) if err != nil { return err } + var statusCode int + if c.inst != nil { + op := c.inst.ExportMetrics(ctx, protoMetrics) + defer func() { op.End(uploadErr, statusCode) }() + } + return errors.Join(uploadErr, c.requestFunc(ctx, func(iCtx context.Context) error { select { case <-iCtx.Done(): @@ -158,6 +177,7 @@ func (c *client) UploadMetrics(ctx context.Context, protoMetrics *metricpb.Resou default: } + statusCode = 0 request.reset(iCtx) // nolint:gosec // URL is constructed from validated OTLP endpoint configuration resp, err := c.httpClient.Do(request.Request) @@ -168,15 +188,18 @@ func (c *client) UploadMetrics(ctx context.Context, protoMetrics *metricpb.Resou if err != nil { return err } - if resp != nil && resp.Body != nil { - defer func() { - if err := resp.Body.Close(); err != nil { - uploadErr = errors.Join(uploadErr, err) - } - }() + if resp != nil { + statusCode = resp.StatusCode + if resp.Body != nil { + defer func() { + if err := resp.Body.Close(); err != nil { + uploadErr = errors.Join(uploadErr, err) + } + }() + } } - if sc := resp.StatusCode; sc >= 200 && sc <= 299 { + if statusCode >= 200 && statusCode <= 299 { // Success, do not retry. // Read the partial success message, if any. @@ -265,7 +288,10 @@ func (c *client) newRequest(ctx context.Context, body []byte) (request, error) { r.Header.Set("Content-Encoding", "gzip") gz := gzPool.Get().(*gzip.Writer) - defer gzPool.Put(gz) + defer func() { + gz.Reset(io.Discard) + gzPool.Put(gz) + }() var b bytes.Buffer gz.Reset(&b) @@ -279,7 +305,7 @@ func (c *client) newRequest(ctx context.Context, body []byte) (request, error) { } req.bodyReader = bodyReader(b.Bytes()) - req.GetBody = bodyReaderErr(body) + req.GetBody = bodyReaderErr(b.Bytes()) } return req, nil diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/config.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/config.go index 2b144f7eb..c6175507a 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/config.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/config.go @@ -185,6 +185,16 @@ func WithTimeout(duration time.Duration) Option { return wrappedOption{oconf.WithTimeout(duration)} } +// WithMaxRequestSize sets the maximum size, in bytes, of a serialized export +// request, before compression, that the exporter will send. +// +// If size is less than or equal to zero, no request-size limit is applied. +// Disabling the limit is not recommended because it can lead to excessive +// resource consumption or abuse. +func WithMaxRequestSize(size int) Option { + return wrappedOption{oconf.WithMaxRequestSize(size)} +} + // WithRetry sets the retry policy for transient retryable errors that are // returned by the target endpoint. // diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter/counter.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter/counter.go new file mode 100644 index 000000000..267cb03d9 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter/counter.go @@ -0,0 +1,31 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/counter/counter.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package counter provides a simple counter for generating unique IDs. +// +// This package is used to generate unique IDs while allowing testing packages +// to reset the counter. +package counter // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter" + +import "sync/atomic" + +// exporterN is a global 0-based count of the number of exporters created. +var exporterN atomic.Int64 + +// NextExporterID returns the next unique ID for an exporter. +func NextExporterID() int64 { + const inc = 1 + return exporterN.Add(inc) - inc +} + +// SetExporterID sets the exporter ID counter to v and returns the previous +// value. +// +// This function is useful for testing purposes, allowing you to reset the +// counter. It should not be used in production code. +func SetExporterID(v int64) int64 { + return exporterN.Swap(v) +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/gen.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/gen.go index a8b709268..f3d165a57 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/gen.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/gen.go @@ -30,3 +30,9 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/o //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/error_test.go.tmpl "--data={}" --out=transform/error_test.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/metricdata.go.tmpl "--data={}" --out=transform/metricdata.go //go:generate gotmpl --body=../../../../../internal/shared/otlp/otlpmetric/transform/metricdata_test.go.tmpl "--data={}" --out=transform/metricdata_test.go + +//go:generate gotmpl --body=../../../../../internal/shared/counter/counter.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/counter\" }" --out=counter/counter.go +//go:generate gotmpl --body=../../../../../internal/shared/counter/counter_test.go.tmpl "--data={}" --out=counter/counter_test.go + +//go:generate gotmpl --body=../../../../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x\" }" --out=x/x.go +//go:generate gotmpl --body=../../../../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/count.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/count.go new file mode 100644 index 000000000..8b5bb1faa --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/count.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package observ // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ" + +import metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + +// countDataPoints counts the total number of data points in a ResourceMetrics. +func countDataPoints(rm *metricpb.ResourceMetrics) int64 { + if rm == nil { + return 0 + } + + var total int64 + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch data := m.Data.(type) { + case *metricpb.Metric_Gauge: + if data.Gauge != nil { + total += int64(len(data.Gauge.DataPoints)) + } + case *metricpb.Metric_Sum: + if data.Sum != nil { + total += int64(len(data.Sum.DataPoints)) + } + case *metricpb.Metric_Histogram: + if data.Histogram != nil { + total += int64(len(data.Histogram.DataPoints)) + } + case *metricpb.Metric_ExponentialHistogram: + if data.ExponentialHistogram != nil { + total += int64(len(data.ExponentialHistogram.DataPoints)) + } + case *metricpb.Metric_Summary: + if data.Summary != nil { + total += int64(len(data.Summary.DataPoints)) + } + } + } + } + return total +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/instrumentation.go new file mode 100644 index 000000000..5db984a23 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ/instrumentation.go @@ -0,0 +1,411 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package observ provides experimental observability instrumentation for the +// otlpmetrichttp exporter. +package observ // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ" + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/netip" + "strconv" + "strings" + "sync" + "time" + + metricpb "go.opentelemetry.io/proto/otlp/metrics/v1" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x" + "go.opentelemetry.io/otel/internal/global" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" +) + +const ( + // ScopeName is the unique name of the meter used for instrumentation. + ScopeName = "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/observ" + + // SchemaURL is the schema URL of the metrics produced by this + // instrumentation. + SchemaURL = semconv.SchemaURL + + // Version is the current version of this instrumentation. + // + // This matches the version of the exporter. + Version = internal.Version +) + +var ( + measureAttrsPool = &sync.Pool{ + New: func() any { + const n = 1 + // component.name + 1 + // component.type + 1 + // server.addr + 1 + // server.port + 1 + // error.type + 1 // http.response.status_code + s := make([]attribute.KeyValue, 0, n) + // Return a pointer to a slice instead of a slice itself + // to avoid allocations on every call. + return &s + }, + } + + addOptPool = &sync.Pool{ + New: func() any { + const n = 1 // WithAttributeSet + o := make([]metric.AddOption, 0, n) + return &o + }, + } + + recordOptPool = &sync.Pool{ + New: func() any { + const n = 1 // WithAttributeSet + o := make([]metric.RecordOption, 0, n) + return &o + }, + } +) + +func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } + +func put[T any](p *sync.Pool, s *[]T) { + *s = (*s)[:0] // Reset. + p.Put(s) +} + +// ComponentName returns the component name for the exporter with the +// provided ID. +func ComponentName(id int64) string { + t := semconv.OTelComponentTypeOtlpHTTPMetricExporter.Value.AsString() + return fmt.Sprintf("%s/%d", t, id) +} + +// Instrumentation is experimental instrumentation for the exporter. +type Instrumentation struct { + inflightMetric metric.Int64UpDownCounter + exportedMetric metric.Int64Counter + opDuration metric.Float64Histogram + + attrs []attribute.KeyValue + addOpt metric.AddOption + recOpt metric.RecordOption +} + +// NewInstrumentation returns instrumentation for an OTLP over HTTP metric +// exporter with the provided ID and endpoint. It uses the global +// MeterProvider to create the instrumentation. +// +// The id should be the unique exporter instance ID. It is used +// to set the "component.name" attribute. +// +// The endpoint is the HTTP endpoint the exporter is exporting to. +// +// If the experimental observability is disabled, nil is returned. +func NewInstrumentation(id int64, endpoint string) (*Instrumentation, error) { + if !x.Observability.Enabled() { + return nil, nil + } + + attrs := BaseAttrs(id, endpoint) + i := &Instrumentation{ + attrs: attrs, + addOpt: metric.WithAttributeSet(attribute.NewSet(attrs...)), + + // Do not modify attrs (NewSet sorts in-place), make a new slice. + recOpt: metric.WithAttributeSet(attribute.NewSet(append( + // Default to OK status code (200). + []attribute.KeyValue{semconv.HTTPResponseStatusCode(http.StatusOK)}, + attrs..., + )...)), + } + + mp := otel.GetMeterProvider() + m := mp.Meter( + ScopeName, + metric.WithInstrumentationVersion(Version), + metric.WithSchemaURL(SchemaURL), + ) + + var err error + + inflightMetric, e := otelconv.NewSDKExporterMetricDataPointInflight(m) + if e != nil { + e = fmt.Errorf("failed to create inflight metric: %w", e) + err = errors.Join(err, e) + } + i.inflightMetric = inflightMetric.Inst() + + exportedMetric, e := otelconv.NewSDKExporterMetricDataPointExported(m) + if e != nil { + e = fmt.Errorf("failed to create exported metric: %w", e) + err = errors.Join(err, e) + } + i.exportedMetric = exportedMetric.Inst() + + opDuration, e := otelconv.NewSDKExporterOperationDuration(m) + if e != nil { + e = fmt.Errorf("failed to create operation duration metric: %w", e) + err = errors.Join(err, e) + } + i.opDuration = opDuration.Inst() + + return i, err +} + +// BaseAttrs returns the base attributes for the exporter with the provided ID +// and endpoint. +// +// The id should be the unique exporter instance ID. It is used +// to set the "component.name" attribute. +// +// The endpoint is the HTTP endpoint the exporter is exporting to. It should be +// in the format "host[:port]". +func BaseAttrs(id int64, endpoint string) []attribute.KeyValue { + host, port, err := parseEndpoint(endpoint) + if err != nil || (host == "" && port < 0) { + if err != nil { + global.Debug("failed to parse endpoint", "endpoint", endpoint, "error", err) + } + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeOtlpHTTPMetricExporter, + } + } + + // Do not use append so the slice is exactly allocated. + + if port < 0 { + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeOtlpHTTPMetricExporter, + semconv.ServerAddress(host), + } + } + + if host == "" { + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeOtlpHTTPMetricExporter, + semconv.ServerPort(port), + } + } + + return []attribute.KeyValue{ + semconv.OTelComponentName(ComponentName(id)), + semconv.OTelComponentTypeOtlpHTTPMetricExporter, + semconv.ServerAddress(host), + semconv.ServerPort(port), + } +} + +// parseEndpoint parses the host and port from endpoint that has the form +// "host[:port]", or it returns an error if the endpoint is not parsable. +// +// If no port is specified, -1 is returned. +// +// If no host is specified, an empty string is returned. +func parseEndpoint(endpoint string) (string, int, error) { + // First check if the endpoint is just an IP address. + if ip := parseIP(endpoint); ip != "" { + return ip, -1, nil + } + + // If there's no colon, there is no port (IPv6 with no port checked above). + if !strings.Contains(endpoint, ":") { + return endpoint, -1, nil + } + + // Otherwise, parse as host:port. + host, portStr, err := net.SplitHostPort(endpoint) + if err != nil { + return "", -1, fmt.Errorf("invalid host:port %q: %w", endpoint, err) + } + + const base, bitSize = 10, 16 + port16, err := strconv.ParseUint(portStr, base, bitSize) + if err != nil { + return "", -1, fmt.Errorf("invalid port %q: %w", portStr, err) + } + port := int(port16) // port is guaranteed to be in the range [0, 65535]. + + return host, port, nil +} + +// parseIP attempts to parse the entire endpoint as an IP address. +// It returns the normalized string form of the IP if successful, +// or an empty string if parsing fails. +func parseIP(ip string) string { + // Strip leading and trailing brackets for IPv6 addresses. + if len(ip) >= 2 && ip[0] == '[' && ip[len(ip)-1] == ']' { + ip = ip[1 : len(ip)-1] + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return "" + } + // Return the normalized string form of the IP. + return addr.String() +} + +// ExportMetrics instruments the UploadMetrics method of the client. It returns an +// [ExportOp] that must have its [ExportOp.End] method called when the +// operation ends. +func (i *Instrumentation) ExportMetrics(ctx context.Context, rm *metricpb.ResourceMetrics) ExportOp { + start := time.Now() + + nMetrics := countDataPoints(rm) + + if i.inflightMetric.Enabled(ctx) { + addOpt := get[metric.AddOption](addOptPool) + defer put(addOptPool, addOpt) + *addOpt = append(*addOpt, i.addOpt) + i.inflightMetric.Add(ctx, nMetrics, *addOpt...) + } + + return ExportOp{ + ctx: ctx, + start: start, + nMetrics: nMetrics, + inst: i, + } +} + +// ExportOp tracks the export operation being observed by +// [Instrumentation.ExportMetrics]. +type ExportOp struct { + ctx context.Context + start time.Time + nMetrics int64 + + inst *Instrumentation +} + +// End completes the observation of the operation being observed by a call to +// [Instrumentation.ExportMetrics]. +// +// Any error that is encountered is provided as err. +// The HTTP status code from the response is provided as status. +// +// If err is not nil, all metrics will be recorded as failures unless error is of +// type [internal.PartialSuccess]. In the case of a PartialSuccess, the number +// of successfully exported metrics will be determined by inspecting the +// RejectedItems field of the PartialSuccess. +func (e ExportOp) End(err error, status int) { + addOpt := get[metric.AddOption](addOptPool) + defer put(addOptPool, addOpt) + *addOpt = append(*addOpt, e.inst.addOpt) + + if e.inst.inflightMetric.Enabled(e.ctx) { + e.inst.inflightMetric.Add(e.ctx, -e.nMetrics, *addOpt...) + } + + success := successful(e.nMetrics, err) + // Record successfully exported metrics, even if the value is 0 which are + // meaningful to distribution aggregations. + if e.inst.exportedMetric.Enabled(e.ctx) { + e.inst.exportedMetric.Add(e.ctx, success, *addOpt...) + } + + if err != nil && e.inst.exportedMetric.Enabled(e.ctx) { + attrs := get[attribute.KeyValue](measureAttrsPool) + defer put(measureAttrsPool, attrs) + *attrs = append(*attrs, e.inst.attrs...) + *attrs = append(*attrs, semconv.ErrorType(err)) + + // Do not inefficiently make a copy of attrs by using + // WithAttributes instead of WithAttributeSet. + o := metric.WithAttributeSet(attribute.NewSet(*attrs...)) + // Reset addOpt with new attribute set. + *addOpt = append((*addOpt)[:0], o) + + e.inst.exportedMetric.Add(e.ctx, e.nMetrics-success, *addOpt...) + } + + if e.inst.opDuration.Enabled(e.ctx) { + recOpt := get[metric.RecordOption](recordOptPool) + defer put(recordOptPool, recOpt) + *recOpt = append(*recOpt, e.inst.recordOption(err, status)) + + d := time.Since(e.start).Seconds() + e.inst.opDuration.Record(e.ctx, d, *recOpt...) + } +} + +// recordOption returns a RecordOption with attributes representing the +// outcome of the operation being recorded. +// +// If err is nil and status is 200, the default recOpt of the +// Instrumentation is returned. +// +// Otherwise, a new RecordOption is returned with the base attributes of the +// Instrumentation plus the http.response.status_code attribute set to the +// provided status (if non-zero), and if err is not nil, the error.type attribute set +// to the type of the error. +func (i *Instrumentation) recordOption(err error, status int) metric.RecordOption { + if err == nil && status == http.StatusOK { + return i.recOpt + } + + attrs := get[attribute.KeyValue](measureAttrsPool) + defer put(measureAttrsPool, attrs) + *attrs = append(*attrs, i.attrs...) + + if status != 0 { + *attrs = append(*attrs, semconv.HTTPResponseStatusCode(status)) + } + if err != nil { + *attrs = append(*attrs, semconv.ErrorType(err)) + } + + // Do not inefficiently make a copy of attrs by using WithAttributes + // instead of WithAttributeSet. + return metric.WithAttributeSet(attribute.NewSet(*attrs...)) +} + +// successful returns the number of successfully exported metrics out of the n +// that were exported based on the provided error. +// +// If err is nil, n is returned. All metrics were successfully exported. +// +// If err is not nil and not an [internal.PartialSuccess] error, 0 is returned. +// It is assumed all metrics failed to be exported. +// +// If err is an [internal.PartialSuccess] error, the number of successfully +// exported metrics is computed by subtracting the RejectedItems field from n. If +// RejectedItems is negative, n is returned. If RejectedItems is greater than +// n, 0 is returned. +func successful(n int64, err error) int64 { + if err == nil { + return n // All metrics successfully exported. + } + // Split rejection calculation so successful is inlinable. + return n - rejected(n, err) +} + +var errPartialPool = &sync.Pool{ + New: func() any { return new(internal.PartialSuccess) }, +} + +// rejected returns how many out of the n metrics were rejected based on the +// provided non-nil err. +func rejected(n int64, err error) int64 { + ps := errPartialPool.Get().(*internal.PartialSuccess) + defer errPartialPool.Put(ps) + // Check for partial success. + if errors.As(err, ps) { + // Bound RejectedItems to [0, n]. This should not be needed, + // but be defensive as this is from an external source. + return min(max(ps.RejectedItems, 0), n) + } + return n // All metrics rejected. +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/oconf/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/oconf/options.go index ed66bb068..3fa7cbe5f 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/oconf/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/oconf/options.go @@ -35,6 +35,9 @@ const ( // DefaultMetricsPath is a default URL path for endpoint that // receives metrics. DefaultMetricsPath string = "/v1/metrics" + // DefaultMaxRequestSize is the default maximum size of a serialized export + // request, before compression. + DefaultMaxRequestSize int = 64 * 1024 * 1024 // DefaultBackoff is a default base backoff time used in the // exponential backoff strategy. DefaultBackoff time.Duration = 300 * time.Millisecond @@ -49,13 +52,14 @@ type ( HTTPTransportProxyFunc func(*http.Request) (*url.URL, error) SignalConfig struct { - Endpoint string - Insecure bool - TLSCfg *tls.Config - Headers map[string]string - Compression Compression - Timeout time.Duration - URLPath string + Endpoint string + Insecure bool + TLSCfg *tls.Config + Headers map[string]string + Compression Compression + MaxRequestSize int + Timeout time.Duration + URLPath string TemporalitySelector metric.TemporalitySelector AggregationSelector metric.AggregationSelector @@ -87,10 +91,11 @@ type ( func NewHTTPConfig(opts ...HTTPOption) Config { cfg := Config{ Metrics: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), - URLPath: DefaultMetricsPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), + URLPath: DefaultMetricsPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, TemporalitySelector: metric.DefaultTemporalitySelector, AggregationSelector: metric.DefaultAggregationSelector, @@ -123,10 +128,11 @@ func cleanPath(urlPath string, defaultPath string) string { func NewGRPCConfig(opts ...GRPCOption) Config { cfg := Config{ Metrics: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), - URLPath: DefaultMetricsPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), + URLPath: DefaultMetricsPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, TemporalitySelector: metric.DefaultTemporalitySelector, AggregationSelector: metric.DefaultAggregationSelector, @@ -354,6 +360,13 @@ func WithTimeout(duration time.Duration) GenericOption { }) } +func WithMaxRequestSize(size int) GenericOption { + return newGenericOption(func(cfg Config) Config { + cfg.Metrics.MaxRequestSize = size + return cfg + }) +} + func WithTemporalitySelector(selector metric.TemporalitySelector) GenericOption { return newGenericOption(func(cfg Config) Config { cfg.Metrics.TemporalitySelector = selector diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/transform/attribute.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/transform/attribute.go index 9e3d8da1e..6a91ff2af 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/transform/attribute.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/transform/attribute.go @@ -81,6 +81,16 @@ func Value(v attribute.Value) *cpb.AnyValue { av.Value = &cpb.AnyValue_StringValue{ StringValue: v.AsString(), } + case attribute.BYTESLICE: + av.Value = &cpb.AnyValue_BytesValue{ + BytesValue: v.AsByteSlice(), + } + case attribute.SLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: attrValues(v.AsSlice()), + }, + } case attribute.STRINGSLICE: av.Value = &cpb.AnyValue_ArrayValue{ ArrayValue: &cpb.ArrayValue{ @@ -143,3 +153,11 @@ func stringSliceValues(vals []string) []*cpb.AnyValue { } return converted } + +func attrValues(vals []attribute.Value) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = Value(v) + } + return converted +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/version.go new file mode 100644 index 000000000..103ac79d0 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/version.go @@ -0,0 +1,8 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal" + +// Version is the current release version of the OpenTelemetry OTLP HTTP metric +// exporter in use. +const Version = "1.44.0" diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/README.md b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/README.md new file mode 100644 index 000000000..ef239cbae --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/README.md @@ -0,0 +1,36 @@ +# Experimental Features + +The `otlpmetrichttp` exporter contains features that have not yet stabilized in the OpenTelemetry specification. +These features are added to the `otlpmetrichttp` exporter prior to stabilization in the specification so that users can start experimenting with them and provide feedback. + +These features may change in backwards incompatible ways as feedback is applied. +See the [Compatibility and Stability](#compatibility-and-stability) section for more information. + +## Features + +- [Observability](#observability) + +### Observability + +The `otlpmetrichttp` exporter can be configured to provide observability about itself using OpenTelemetry metrics. + +To opt-in, set the environment variable `OTEL_GO_X_OBSERVABILITY` to `true`. + +When enabled, the exporter will create the following metrics using the global `MeterProvider`: + +- `otel.sdk.exporter.metric_data_point.inflight` +- `otel.sdk.exporter.metric_data_point.exported` +- `otel.sdk.exporter.operation.duration` + +Please see the [Semantic conventions for OpenTelemetry SDK metrics] documentation for more details on these metrics. + +[Semantic conventions for OpenTelemetry SDK metrics]: https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/otel/sdk-metrics.md + +## Compatibility and Stability + +Experimental features do not fall within the scope of the OpenTelemetry Go versioning and stability [policy](../../../../../../VERSIONING.md). +These features may be removed or modified in successive version releases, including patch versions. + +When an experimental feature is promoted to a stable feature, a migration path will be included in the changelog entry of the release. +There is no guarantee that any environment variable feature flags that enabled the experimental feature will be supported by the stable version. +If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/observ.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/observ.go new file mode 100644 index 000000000..296946a3b --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/observ.go @@ -0,0 +1,22 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package x // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x" + +import "strings" + +// Observability is an experimental feature flag that determines if exporter +// observability metrics are enabled. +// +// To enable this feature set the OTEL_GO_X_OBSERVABILITY environment variable +// to the case-insensitive string value of "true" (i.e. "True" and "TRUE" +// will also enable this). +var Observability = newFeature( + []string{"OBSERVABILITY"}, + func(v string) (string, bool) { + if strings.EqualFold(v, "true") { + return v, true + } + return "", false + }, +) diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/x.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/x.go new file mode 100644 index 000000000..663375f63 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x/x.go @@ -0,0 +1,58 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/x/x.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package x documents experimental features for [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x]. +package x // import "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp/internal/x" + +import ( + "os" +) + +// Feature is an experimental feature control flag. It provides a uniform way +// to interact with these feature flags and parse their values. +type Feature[T any] struct { + keys []string + parse func(v string) (T, bool) +} + +func newFeature[T any](suffix []string, parse func(string) (T, bool)) Feature[T] { + const envKeyRoot = "OTEL_GO_X_" + keys := make([]string, 0, len(suffix)) + for _, s := range suffix { + keys = append(keys, envKeyRoot+s) + } + return Feature[T]{ + keys: keys, + parse: parse, + } +} + +// Keys returns the environment variable keys that can be set to enable the +// feature. +func (f Feature[T]) Keys() []string { return f.keys } + +// Lookup returns the user configured value for the feature and true if the +// user has enabled the feature. Otherwise, if the feature is not enabled, a +// zero-value and false are returned. +func (f Feature[T]) Lookup() (v T, ok bool) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/62effed618589a0bec416a87e559c0a9d96289bb/specification/configuration/sdk-environment-variables.md#parsing-empty-value + // + // > The SDK MUST interpret an empty value of an environment variable the + // > same way as when the variable is unset. + for _, key := range f.keys { + vRaw := os.Getenv(key) + if vRaw != "" { + return f.parse(vRaw) + } + } + return v, ok +} + +// Enabled reports whether the feature is enabled. +func (f Feature[T]) Enabled() bool { + _, ok := f.Lookup() + return ok +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform/attribute.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform/attribute.go index 12e243e04..0d43a5dc5 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform/attribute.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform/attribute.go @@ -87,6 +87,16 @@ func Value(v attribute.Value) *commonpb.AnyValue { av.Value = &commonpb.AnyValue_StringValue{ StringValue: v.AsString(), } + case attribute.BYTESLICE: + av.Value = &commonpb.AnyValue_BytesValue{ + BytesValue: v.AsByteSlice(), + } + case attribute.SLICE: + av.Value = &commonpb.AnyValue_ArrayValue{ + ArrayValue: &commonpb.ArrayValue{ + Values: values(v.AsSlice()), + }, + } case attribute.STRINGSLICE: av.Value = &commonpb.AnyValue_ArrayValue{ ArrayValue: &commonpb.ArrayValue{ @@ -149,3 +159,11 @@ func stringSliceValues(vals []string) []*commonpb.AnyValue { } return converted } + +func values(vals []attribute.Value) []*commonpb.AnyValue { + converted := make([]*commonpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = Value(v) + } + return converted +} diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/client.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/client.go index 258d0ca6a..e8b33d3fa 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/client.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/client.go @@ -6,6 +6,7 @@ package otlptracegrpc // import "go.opentelemetry.io/otel/exporters/otlp/otlptra import ( "context" "errors" + "fmt" "sync" "time" @@ -16,6 +17,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal" @@ -26,11 +28,12 @@ import ( ) type client struct { - endpoint string - dialOpts []grpc.DialOption - metadata metadata.MD - exportTimeout time.Duration - requestFunc retry.RequestFunc + endpoint string + dialOpts []grpc.DialOption + metadata metadata.MD + exportTimeout time.Duration + maxRequestSize int + requestFunc retry.RequestFunc // stopCtx is used as a parent context for all exports. Therefore, when it // is canceled with the stopFunc all exports are canceled. @@ -65,14 +68,15 @@ func newClient(opts ...Option) *client { ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel called in client shutdown. c := &client{ - endpoint: cfg.Traces.Endpoint, - exportTimeout: cfg.Traces.Timeout, - requestFunc: cfg.RetryConfig.RequestFunc(retryable), - dialOpts: cfg.DialOptions, - stopCtx: ctx, - stopFunc: cancel, - conn: cfg.GRPCConn, - instID: counter.NextExporterID(), + endpoint: cfg.Traces.Endpoint, + exportTimeout: cfg.Traces.Timeout, + maxRequestSize: cfg.Traces.MaxRequestSize, + requestFunc: cfg.RetryConfig.RequestFunc(retryable), + dialOpts: cfg.DialOptions, + stopCtx: ctx, + stopFunc: cancel, + conn: cfg.GRPCConn, + instID: counter.NextExporterID(), } if len(cfg.Traces.Headers) > 0 { @@ -205,16 +209,28 @@ func (c *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc ctx, cancel := c.exportContext(ctx) defer cancel() - var code codes.Code + pbRequest := &coltracepb.ExportTraceServiceRequest{ + ResourceSpans: protoSpans, + } + + code := codes.Unknown if c.inst != nil { - op := c.inst.ExportSpans(ctx, len(protoSpans)) + var spanCount int + for _, rs := range protoSpans { + for _, ss := range rs.ScopeSpans { + spanCount += len(ss.Spans) + } + } + op := c.inst.ExportSpans(ctx, spanCount) defer func() { op.End(uploadErr, code) }() } + if maxSize := c.maxRequestSize; maxSize > 0 && proto.Size(pbRequest) > maxSize { + return fmt.Errorf("request message too large: exceeded %d bytes", maxSize) + } + return c.requestFunc(ctx, func(iCtx context.Context) error { - resp, err := c.tsc.Export(iCtx, &coltracepb.ExportTraceServiceRequest{ - ResourceSpans: protoSpans, - }) + resp, err := c.tsc.Export(iCtx, pbRequest) if resp != nil && resp.PartialSuccess != nil { msg := resp.PartialSuccess.GetErrorMessage() n := resp.PartialSuccess.GetRejectedSpans() diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/observ/instrumentation.go index a84733174..676a93514 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/observ/instrumentation.go @@ -18,8 +18,8 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/x" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( @@ -72,6 +72,7 @@ var ( func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } func put[T any](p *sync.Pool, s *[]T) { + clear(*s) // erase elements to allow GC to collect what they refer to. *s = (*s)[:0] // Reset. p.Put(s) } @@ -339,7 +340,10 @@ var errPartialPool = &sync.Pool{ // the provided non-nil err. func rejected(n int64, err error) int64 { ps := errPartialPool.Get().(*internal.PartialSuccess) - defer errPartialPool.Put(ps) + defer func() { + *ps = internal.PartialSuccess{} // erase fields to allow GC to collect them. + errPartialPool.Put(ps) + }() // Check for partial success. if errors.As(err, ps) { // Bound RejectedItems to [0, n]. This should not be needed, diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/otlpconfig/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/otlpconfig/options.go index 4f47117a5..d940b6762 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/otlpconfig/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/otlpconfig/options.go @@ -31,6 +31,9 @@ const ( // DefaultTracesPath is a default URL path for endpoint that // receives spans. DefaultTracesPath string = "/v1/traces" + // DefaultMaxRequestSize is the default maximum size of a serialized export + // request, before compression. + DefaultMaxRequestSize int = 64 * 1024 * 1024 // DefaultTimeout is a default max waiting time for the backend to process // each span batch. DefaultTimeout time.Duration = 10 * time.Second @@ -42,13 +45,14 @@ type ( HTTPTransportProxyFunc func(*http.Request) (*url.URL, error) SignalConfig struct { - Endpoint string - Insecure bool - TLSCfg *tls.Config - Headers map[string]string - Compression Compression - Timeout time.Duration - URLPath string + Endpoint string + Insecure bool + TLSCfg *tls.Config + Headers map[string]string + Compression Compression + MaxRequestSize int + Timeout time.Duration + URLPath string // gRPC configurations GRPCCredentials credentials.TransportCredentials @@ -77,10 +81,11 @@ type ( func NewHTTPConfig(opts ...HTTPOption) Config { cfg := Config{ Traces: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), - URLPath: DefaultTracesPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), + URLPath: DefaultTracesPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, }, RetryConfig: retry.DefaultConfig, } @@ -111,10 +116,11 @@ func NewGRPCConfig(opts ...GRPCOption) Config { userAgent := "OTel OTLP Exporter Go/" + otlptrace.Version() cfg := Config{ Traces: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), - URLPath: DefaultTracesPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), + URLPath: DefaultTracesPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, }, RetryConfig: retry.DefaultConfig, DialOptions: []grpc.DialOption{grpc.WithUserAgent(userAgent)}, @@ -345,6 +351,13 @@ func WithTimeout(duration time.Duration) GenericOption { }) } +func WithMaxRequestSize(size int) GenericOption { + return newGenericOption(func(cfg Config) Config { + cfg.Traces.MaxRequestSize = size + return cfg + }) +} + func WithProxy(pf HTTPTransportProxyFunc) GenericOption { return newGenericOption(func(cfg Config) Config { cfg.Traces.Proxy = pf diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/version.go index 7a1c420ec..28e51e443 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/internal/version.go @@ -5,4 +5,4 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlptrace/ot // Version is the current release version of the OpenTelemetry OTLP gRPC trace // exporter in use. -const Version = "1.43.0" +const Version = "1.44.0" diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/options.go index 2da229870..b32006052 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc/options.go @@ -192,6 +192,16 @@ func WithTimeout(duration time.Duration) Option { return wrappedOption{otlpconfig.WithTimeout(duration)} } +// WithMaxRequestSize sets the maximum size, in bytes, of a serialized export +// request, before compression, that the exporter will send. +// +// If size is less than or equal to zero, no request-size limit is applied. +// Disabling the limit is not recommended because it can lead to excessive +// resource consumption or abuse. +func WithMaxRequestSize(size int) Option { + return wrappedOption{otlpconfig.WithMaxRequestSize(size)} +} + // WithRetry sets the retry policy for transient retryable errors that may be // returned by the target endpoint when exporting a batch of spans. // diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go index 4ae569ff4..f81098b38 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/client.go @@ -143,9 +143,9 @@ func (c *client) Start(ctx context.Context) error { } // Stop shuts down the client and interrupt any in-flight request. -func (d *client) Stop(ctx context.Context) error { - d.stopOnce.Do(func() { - close(d.stopCh) +func (c *client) Stop(ctx context.Context) error { + c.stopOnce.Do(func() { + close(c.stopCh) }) select { case <-ctx.Done(): @@ -156,7 +156,7 @@ func (d *client) Stop(ctx context.Context) error { } // UploadTraces sends a batch of spans to the collector. -func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.ResourceSpans) (uploadErr error) { +func (c *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.ResourceSpans) (uploadErr error) { pbRequest := &coltracepb.ExportTraceServiceRequest{ ResourceSpans: protoSpans, } @@ -165,30 +165,41 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc return err } - ctx, cancel := d.contextWithStop(ctx) + ctx, cancel := c.contextWithStop(ctx) defer cancel() - request, err := d.newRequest(rawRequest) + if maxSize := c.cfg.MaxRequestSize; maxSize > 0 && len(rawRequest) > maxSize { + return fmt.Errorf("request body too large: exceeded %d bytes", maxSize) + } + + request, err := c.newRequest(rawRequest) if err != nil { return err } var statusCode int - if d.inst != nil { - op := d.inst.ExportSpans(ctx, len(protoSpans)) + if c.inst != nil { + var spanCount int + for _, rs := range protoSpans { + for _, ss := range rs.ScopeSpans { + spanCount += len(ss.Spans) + } + } + op := c.inst.ExportSpans(ctx, spanCount) defer func() { op.End(uploadErr, statusCode) }() } - return errors.Join(uploadErr, d.requestFunc(ctx, func(ctx context.Context) error { + return errors.Join(uploadErr, c.requestFunc(ctx, func(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() default: } + statusCode = 0 request.reset(ctx) // nolint:gosec // URL is constructed from validated OTLP endpoint configuration - resp, err := d.client.Do(request.Request) + resp, err := c.client.Do(request.Request) var urlErr *url.Error if errors.As(err, &urlErr) && urlErr.Temporary() { return newResponseError(http.Header{}, err) @@ -272,8 +283,8 @@ func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc })) } -func (d *client) newRequest(body []byte) (request, error) { - u := url.URL{Scheme: d.getScheme(), Host: d.cfg.Endpoint, Path: d.cfg.URLPath} +func (c *client) newRequest(body []byte) (request, error) { + u := url.URL{Scheme: c.getScheme(), Host: c.cfg.Endpoint, Path: c.cfg.URLPath} r, err := http.NewRequestWithContext(context.Background(), http.MethodPost, u.String(), http.NoBody) if err != nil { return request{Request: r}, err @@ -282,13 +293,13 @@ func (d *client) newRequest(body []byte) (request, error) { userAgent := "OTel OTLP Exporter Go/" + otlptrace.Version() r.Header.Set("User-Agent", userAgent) - for k, v := range d.cfg.Headers { + for k, v := range c.cfg.Headers { r.Header.Set(k, v) } r.Header.Set("Content-Type", contentTypeProto) req := request{Request: r} - switch Compression(d.cfg.Compression) { + switch Compression(c.cfg.Compression) { case NoCompression: r.ContentLength = int64(len(body)) req.bodyReader = bodyReader(body) @@ -299,7 +310,10 @@ func (d *client) newRequest(body []byte) (request, error) { r.Header.Set("Content-Encoding", "gzip") gz := gzPool.Get().(*gzip.Writer) - defer gzPool.Put(gz) + defer func() { + gz.Reset(io.Discard) + gzPool.Put(gz) + }() var b bytes.Buffer gz.Reset(&b) @@ -320,15 +334,15 @@ func (d *client) newRequest(body []byte) (request, error) { } // MarshalLog is the marshaling function used by the logging system to represent this Client. -func (d *client) MarshalLog() any { +func (c *client) MarshalLog() any { return struct { Type string Endpoint string Insecure bool }{ Type: "otlptracehttp", - Endpoint: d.cfg.Endpoint, - Insecure: d.cfg.Insecure, + Endpoint: c.cfg.Endpoint, + Insecure: c.cfg.Insecure, } } @@ -425,14 +439,14 @@ func evaluate(err error) (bool, time.Duration) { return true, time.Duration(rErr.throttle) } -func (d *client) getScheme() string { - if d.cfg.Insecure { +func (c *client) getScheme() string { + if c.cfg.Insecure { return "http" } return "https" } -func (d *client) contextWithStop(ctx context.Context) (context.Context, context.CancelFunc) { +func (c *client) contextWithStop(ctx context.Context) (context.Context, context.CancelFunc) { // Unify the parent context Done signal with the client's stop // channel. ctx, cancel := context.WithCancel(ctx) @@ -441,7 +455,7 @@ func (d *client) contextWithStop(ctx context.Context) (context.Context, context. case <-ctx.Done(): // Nothing to do, either cancelled or deadline // happened. - case <-d.stopCh: + case <-c.stopCh: cancel() } }(ctx, cancel) diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go index 3f2556e7a..1f4fc55c6 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ/instrumentation.go @@ -23,8 +23,8 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/x" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( @@ -77,6 +77,7 @@ var ( func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } func put[T any](p *sync.Pool, s *[]T) { + clear(*s) // erase elements to allow GC to collect what they refer to. *s = (*s)[:0] // Reset. p.Put(s) } @@ -167,7 +168,7 @@ func NewInstrumentation(id int64, endpoint string) (*Instrumentation, error) { // to set the "component.name" attribute. // // The endpoint is the HTTP endpoint the exporter is exporting to. It should be -// in the format "host:port" or a full URL. +// in the format "host[:port]". func BaseAttrs(id int64, endpoint string) []attribute.KeyValue { host, port, err := parseEndpoint(endpoint) if err != nil || (host == "" && port < 0) { @@ -345,7 +346,7 @@ func (e ExportOp) End(err error, status int) { // // Otherwise, a new RecordOption is returned with the base attributes of the // Instrumentation plus the http.response.status_code attribute set to the -// provided status, and if err is not nil, the error.type attribute set +// provided status (if non-zero), and if err is not nil, the error.type attribute set // to the type of the error. func (i *Instrumentation) recordOption(err error, status int) metric.RecordOption { if err == nil && status == http.StatusOK { @@ -356,7 +357,9 @@ func (i *Instrumentation) recordOption(err error, status int) metric.RecordOptio defer put(measureAttrsPool, attrs) *attrs = append(*attrs, i.attrs...) - *attrs = append(*attrs, semconv.HTTPResponseStatusCode(status)) + if status != 0 { + *attrs = append(*attrs, semconv.HTTPResponseStatusCode(status)) + } if err != nil { *attrs = append(*attrs, semconv.ErrorType(err)) } @@ -394,7 +397,10 @@ var errPartialPool = &sync.Pool{ // the provided non-nil err. func rejected(n int64, err error) int64 { ps := errPartialPool.Get().(*internal.PartialSuccess) - defer errPartialPool.Put(ps) + defer func() { + *ps = internal.PartialSuccess{} // erase fields to allow GC to collect them. + errPartialPool.Put(ps) + }() // Check for partial success. if errors.As(err, ps) { // Bound RejectedItems to [0, n]. This should not be needed, diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig/options.go index e415feea6..2abde6615 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig/options.go @@ -31,6 +31,9 @@ const ( // DefaultTracesPath is a default URL path for endpoint that // receives spans. DefaultTracesPath string = "/v1/traces" + // DefaultMaxRequestSize is the default maximum size of a serialized export + // request, before compression. + DefaultMaxRequestSize int = 64 * 1024 * 1024 // DefaultTimeout is a default max waiting time for the backend to process // each span batch. DefaultTimeout time.Duration = 10 * time.Second @@ -42,13 +45,14 @@ type ( HTTPTransportProxyFunc func(*http.Request) (*url.URL, error) SignalConfig struct { - Endpoint string - Insecure bool - TLSCfg *tls.Config - Headers map[string]string - Compression Compression - Timeout time.Duration - URLPath string + Endpoint string + Insecure bool + TLSCfg *tls.Config + Headers map[string]string + Compression Compression + MaxRequestSize int + Timeout time.Duration + URLPath string // gRPC configurations GRPCCredentials credentials.TransportCredentials @@ -77,10 +81,11 @@ type ( func NewHTTPConfig(opts ...HTTPOption) Config { cfg := Config{ Traces: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), - URLPath: DefaultTracesPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorHTTPPort), + URLPath: DefaultTracesPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, }, RetryConfig: retry.DefaultConfig, } @@ -111,10 +116,11 @@ func NewGRPCConfig(opts ...GRPCOption) Config { userAgent := "OTel OTLP Exporter Go/" + otlptrace.Version() cfg := Config{ Traces: SignalConfig{ - Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), - URLPath: DefaultTracesPath, - Compression: NoCompression, - Timeout: DefaultTimeout, + Endpoint: fmt.Sprintf("%s:%d", DefaultCollectorHost, DefaultCollectorGRPCPort), + URLPath: DefaultTracesPath, + Compression: NoCompression, + MaxRequestSize: DefaultMaxRequestSize, + Timeout: DefaultTimeout, }, RetryConfig: retry.DefaultConfig, DialOptions: []grpc.DialOption{grpc.WithUserAgent(userAgent)}, @@ -345,6 +351,13 @@ func WithTimeout(duration time.Duration) GenericOption { }) } +func WithMaxRequestSize(size int) GenericOption { + return newGenericOption(func(cfg Config) Config { + cfg.Traces.MaxRequestSize = size + return cfg + }) +} + func WithProxy(pf HTTPTransportProxyFunc) GenericOption { return newGenericOption(func(cfg Config) Config { cfg.Traces.Proxy = pf diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go index 3e43f7711..882b67138 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/version.go @@ -5,4 +5,4 @@ package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlptrace/ot // Version is the current release version of the OpenTelemetry OTLP HTTP trace // exporter in use. -const Version = "1.43.0" +const Version = "1.44.0" diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/options.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/options.go index cfe21dbfb..291b5f9b6 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/options.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/options.go @@ -138,6 +138,16 @@ func WithTimeout(duration time.Duration) Option { return wrappedOption{otlpconfig.WithTimeout(duration)} } +// WithMaxRequestSize sets the maximum size, in bytes, of a serialized export +// request, before compression, that the exporter will send. +// +// If size is less than or equal to zero, no request-size limit is applied. +// Disabling the limit is not recommended because it can lead to excessive +// resource consumption or abuse. +func WithMaxRequestSize(size int) Option { + return wrappedOption{otlpconfig.WithMaxRequestSize(size)} +} + // WithRetry configures the retry policy for transient errors that may occurs // when exporting traces. An exponential back-off algorithm is used to ensure // endpoints are not overwhelmed with retries. If unset, the default retry diff --git a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/version.go b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/version.go index 087e95f7b..d847210db 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/otlp/otlptrace/version.go @@ -5,5 +5,5 @@ package otlptrace // import "go.opentelemetry.io/otel/exporters/otlp/otlptrace" // Version is the current release version of the OpenTelemetry OTLP trace exporter in use. func Version() string { - return "1.43.0" + return "1.44.0" } diff --git a/vendor/go.opentelemetry.io/otel/exporters/prometheus/exporter.go b/vendor/go.opentelemetry.io/otel/exporters/prometheus/exporter.go index 0f2259af3..bc0103893 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/prometheus/exporter.go +++ b/vendor/go.opentelemetry.io/otel/exporters/prometheus/exporter.go @@ -91,14 +91,18 @@ type collector struct { namespace string resourceAttributesFilter attribute.Filter - mu sync.Mutex // mu protects all members below from the concurrent access. + mu sync.Mutex disableTargetInfo bool targetInfo prometheus.Metric metricFamilies map[string]*dto.MetricFamily - resourceKeyVals keyVals - metricNamer otlptranslator.MetricNamer - labelNamer otlptranslator.LabelNamer - unitNamer otlptranslator.UnitNamer + + resourceKeyValsOnce sync.Once + resourceKeyVals keyVals + resourceKeyValsErr error + + metricNamer otlptranslator.MetricNamer + labelNamer otlptranslator.LabelNamer + unitNamer otlptranslator.UnitNamer inst *observ.Instrumentation @@ -178,7 +182,10 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { } metrics := metricsPool.Get().(*metricdata.ResourceMetrics) - defer metricsPool.Put(metrics) + defer func() { + *metrics = metricdata.ResourceMetrics{} // erase fields to allow GC to collect them. + metricsPool.Put(metrics) + }() endCollection := func(error) {} if c.inst != nil { @@ -226,11 +233,13 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { ch <- c.targetInfo } - if c.resourceAttributesFilter != nil && len(c.resourceKeyVals.keys) == 0 { - e := c.createResourceAttributes(metrics.Resource) - if e != nil { - otel.Handle(e) - err = errors.Join(err, fmt.Errorf("failed to createResourceAttributes: %w", e)) + if c.resourceAttributesFilter != nil { + c.resourceKeyValsOnce.Do(func() { + c.resourceKeyVals, c.resourceKeyValsErr = c.createResourceAttributes(metrics.Resource) + }) + if c.resourceKeyValsErr != nil { + otel.Handle(c.resourceKeyValsErr) + err = errors.Join(err, fmt.Errorf("failed to createResourceAttributes: %w", c.resourceKeyValsErr)) return } } @@ -242,7 +251,8 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { }) continue } - n := len(c.resourceKeyVals.keys) + 2 // resource attrs + scope name + scope version + // resource attributes + scope attributes + scope name + scope version + scope schema url + n := len(c.resourceKeyVals.keys) + 3 + scopeMetrics.Scope.Attributes.Len() kv := keyVals{ keys: make([]string, 0, n), vals: make([]string, 0, n), @@ -252,15 +262,12 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { kv.keys = append(kv.keys, scopeNameLabel, scopeVersionLabel, scopeSchemaLabel) kv.vals = append(kv.vals, scopeMetrics.Scope.Name, scopeMetrics.Scope.Version, scopeMetrics.Scope.SchemaURL) - attrKeys, attrVals, e := getAttrs(scopeMetrics.Scope.Attributes, c.labelNamer) + attrKeys, attrVals, e := getScopeAttrs(scopeMetrics.Scope.Attributes, c.labelNamer) if e != nil { reportError(ch, nil, e) - err = errors.Join(err, fmt.Errorf("failed to getAttrs for ScopeMetrics %d: %w", j, e)) + err = errors.Join(err, fmt.Errorf("failed to translate scope attributes for ScopeMetrics %d: %w", j, e)) continue } - for i := range attrKeys { - attrKeys[i] = scopeLabelPrefix + attrKeys[i] - } kv.keys = append(kv.keys, attrKeys...) kv.vals = append(kv.vals, attrVals...) } @@ -451,7 +458,8 @@ func addExponentialHistogramMetric[N int64 | float64]( scale, dp.ZeroThreshold, dp.StartTime, - values...) + values..., + ) if e != nil { reportError(ch, desc, e) err = errors.Join( @@ -617,7 +625,7 @@ func getAttrs(attrs attribute.Set, labelNamer otlptranslator.LabelNamer) ([]stri for itr.Next() { kv := itr.Attribute() keys = append(keys, string(kv.Key)) - values = append(values, kv.Value.Emit()) + values = append(values, kv.Value.String()) } } else { // It sanitizes invalid characters and handles duplicate keys @@ -631,10 +639,10 @@ func getAttrs(attrs attribute.Set, labelNamer otlptranslator.LabelNamer) ([]stri return nil, nil, err } if _, ok := keysMap[key]; !ok { - keysMap[key] = []string{kv.Value.Emit()} + keysMap[key] = []string{kv.Value.String()} } else { // if the sanitized key is a duplicate, append to the list of keys - keysMap[key] = append(keysMap[key], kv.Value.Emit()) + keysMap[key] = append(keysMap[key], kv.Value.String()) } } for key, vals := range keysMap { @@ -646,6 +654,56 @@ func getAttrs(attrs attribute.Set, labelNamer otlptranslator.LabelNamer) ([]stri return keys, values, nil } +func getScopeAttrs(attrs attribute.Set, labelNamer otlptranslator.LabelNamer) ([]string, []string, error) { + keys := make([]string, 0, attrs.Len()) + values := make([]string, 0, attrs.Len()) + itr := attrs.Iter() + + if labelNamer.UTF8Allowed { + for itr.Next() { + kv := itr.Attribute() + key := string(kv.Key) + if isReservedScopeLabel(key) { + continue + } + keys = append(keys, scopeLabelPrefix+key) + values = append(values, kv.Value.String()) + } + return keys, values, nil + } + + keysMap := make(map[string][]string) + for itr.Next() { + kv := itr.Attribute() + key, err := labelNamer.Build(string(kv.Key)) + if err != nil { + // TODO(#7066) Handle this error better. + return nil, nil, err + } + if isReservedScopeLabel(key) { + continue + } + keysMap[key] = append(keysMap[key], kv.Value.String()) + } + + for key, vals := range keysMap { + keys = append(keys, scopeLabelPrefix+key) + slices.Sort(vals) + values = append(values, strings.Join(vals, ";")) + } + + return keys, values, nil +} + +func isReservedScopeLabel(key string) bool { + switch key { + case "name", "version", "schema_url": + return true + default: + return false + } +} + func (c *collector) createInfoMetric(name, description string, res *resource.Resource) (prometheus.Metric, error) { keys, values, err := getAttrs(*res.Set(), c.labelNamer) if err != nil { @@ -719,18 +777,14 @@ func (c *collector) namingMetricType(m metricdata.Metrics) otlptranslator.Metric return otlptranslator.MetricTypeUnknown } -func (c *collector) createResourceAttributes(res *resource.Resource) error { - c.mu.Lock() - defer c.mu.Unlock() - +func (c *collector) createResourceAttributes(res *resource.Resource) (keyVals, error) { resourceAttrs, _ := res.Set().Filter(c.resourceAttributesFilter) resourceKeys, resourceValues, err := getAttrs(resourceAttrs, c.labelNamer) if err != nil { - return err + return keyVals{}, err } - c.resourceKeyVals = keyVals{keys: resourceKeys, vals: resourceValues} - return nil + return keyVals{keys: resourceKeys, vals: resourceValues}, nil } func (c *collector) validateMetrics(name, description string, metricType *dto.MetricType) (drop bool, help string) { @@ -812,7 +866,7 @@ func attributesToLabels(attrs []attribute.KeyValue, labelNamer otlptranslator.La if err != nil { return nil, err } - labels[name] = attr.Value.Emit() + labels[name] = attr.Value.String() } return labels, nil } diff --git a/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/observ/instrumentation.go index bc6f33fe6..c995b982c 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/observ/instrumentation.go @@ -17,8 +17,8 @@ import ( "go.opentelemetry.io/otel/exporters/prometheus/internal" "go.opentelemetry.io/otel/exporters/prometheus/internal/x" "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( @@ -71,6 +71,7 @@ var ( func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } func put[T any](p *sync.Pool, s *[]T) { + clear(*s) // erase elements to allow GC to collect what they refer to. *s = (*s)[:0] // Reset. p.Put(s) } diff --git a/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/version.go index a3527e265..5616fe25a 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/prometheus/internal/version.go @@ -6,4 +6,4 @@ package internal // import "go.opentelemetry.io/otel/exporters/prometheus/intern // Version is the current release version of the OpenTelemetry prometheus // exporter in use. -const Version = "0.65.0" +const Version = "0.66.0" diff --git a/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/observ/instrumentation.go index 901cc938c..08e276786 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/observ/instrumentation.go @@ -17,8 +17,8 @@ import ( "go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/x" "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( @@ -75,6 +75,7 @@ var ( func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } func put[T any](p *sync.Pool, s *[]T) { + clear(*s) *s = (*s)[:0] // Reset. p.Put(s) } diff --git a/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/version.go b/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/version.go index 78e618c47..c22b38fd5 100644 --- a/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/version.go +++ b/vendor/go.opentelemetry.io/otel/exporters/stdout/stdouttrace/internal/version.go @@ -5,4 +5,4 @@ package internal // import "go.opentelemetry.io/otel/exporters/stdout/stdouttrac // Version is the current release version of the OpenTelemetry stdouttrace // exporter in use. -const Version = "1.43.0" +const Version = "1.44.0" diff --git a/vendor/go.opentelemetry.io/otel/metric/asyncfloat64.go b/vendor/go.opentelemetry.io/otel/metric/asyncfloat64.go index 466812d34..1d21e2eb7 100644 --- a/vendor/go.opentelemetry.io/otel/metric/asyncfloat64.go +++ b/vendor/go.opentelemetry.io/otel/metric/asyncfloat64.go @@ -51,6 +51,9 @@ type Float64ObservableCounterConfig struct { func NewFloat64ObservableCounterConfig(opts ...Float64ObservableCounterOption) Float64ObservableCounterConfig { var config Float64ObservableCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64ObservableCounter(config) } return config @@ -111,6 +114,9 @@ func NewFloat64ObservableUpDownCounterConfig( ) Float64ObservableUpDownCounterConfig { var config Float64ObservableUpDownCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64ObservableUpDownCounter(config) } return config @@ -168,6 +174,9 @@ type Float64ObservableGaugeConfig struct { func NewFloat64ObservableGaugeConfig(opts ...Float64ObservableGaugeOption) Float64ObservableGaugeConfig { var config Float64ObservableGaugeConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64ObservableGauge(config) } return config diff --git a/vendor/go.opentelemetry.io/otel/metric/asyncint64.go b/vendor/go.opentelemetry.io/otel/metric/asyncint64.go index 66c971bd8..9d45a4d41 100644 --- a/vendor/go.opentelemetry.io/otel/metric/asyncint64.go +++ b/vendor/go.opentelemetry.io/otel/metric/asyncint64.go @@ -50,6 +50,9 @@ type Int64ObservableCounterConfig struct { func NewInt64ObservableCounterConfig(opts ...Int64ObservableCounterOption) Int64ObservableCounterConfig { var config Int64ObservableCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64ObservableCounter(config) } return config @@ -110,6 +113,9 @@ func NewInt64ObservableUpDownCounterConfig( ) Int64ObservableUpDownCounterConfig { var config Int64ObservableUpDownCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64ObservableUpDownCounter(config) } return config @@ -167,6 +173,9 @@ type Int64ObservableGaugeConfig struct { func NewInt64ObservableGaugeConfig(opts ...Int64ObservableGaugeOption) Int64ObservableGaugeConfig { var config Int64ObservableGaugeConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64ObservableGauge(config) } return config diff --git a/vendor/go.opentelemetry.io/otel/metric/config.go b/vendor/go.opentelemetry.io/otel/metric/config.go index e42dd6e70..889545e23 100644 --- a/vendor/go.opentelemetry.io/otel/metric/config.go +++ b/vendor/go.opentelemetry.io/otel/metric/config.go @@ -42,11 +42,18 @@ type MeterOption interface { applyMeter(MeterConfig) MeterConfig } +type experimentalOption interface { + Experimental() +} + // NewMeterConfig creates a new MeterConfig and applies // all the given options. func NewMeterConfig(opts ...MeterOption) MeterConfig { var config MeterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyMeter(config) } return config diff --git a/vendor/go.opentelemetry.io/otel/metric/doc.go b/vendor/go.opentelemetry.io/otel/metric/doc.go index f153745b0..794e1a8ba 100644 --- a/vendor/go.opentelemetry.io/otel/metric/doc.go +++ b/vendor/go.opentelemetry.io/otel/metric/doc.go @@ -24,10 +24,10 @@ all instruments fall into two overlapping logical categories: asynchronous or synchronous, and int64 or float64. All synchronous instruments ([Int64Counter], [Int64UpDownCounter], -[Int64Histogram], [Float64Counter], [Float64UpDownCounter], and -[Float64Histogram]) are used to measure the operation and performance of source -code during the source code execution. These instruments only make measurements -when the source code they instrument is run. +[Int64Histogram], [Int64Gauge], [Float64Counter], [Float64UpDownCounter], +[Float64Histogram], and [Float64Gauge]) are used to measure the operation and +performance of source code during the source code execution. These instruments +only make measurements when the source code they instrument is run. All asynchronous instruments ([Int64ObservableCounter], [Int64ObservableUpDownCounter], [Int64ObservableGauge], @@ -50,9 +50,11 @@ incrementally increase in value. UpDownCounters ([Int64UpDownCounter], values that can increase and decrease. When more information needs to be conveyed about all the synchronous measurements made during a collection cycle, a Histogram ([Int64Histogram] and [Float64Histogram]) should be used. Finally, -when just the most recent measurement needs to be conveyed about an -asynchronous measurement, a Gauge ([Int64ObservableGauge] and -[Float64ObservableGauge]) should be used. +when just the most recent measurement needs to be conveyed, a Gauge +([Int64Gauge], [Float64Gauge], [Int64ObservableGauge], and +[Float64ObservableGauge]) should be used: the synchronous variants record an +instantaneous value at a specific point in code, while the observable variants +sample the value via a callback once per collection cycle. See the [OpenTelemetry documentation] for more information about instruments and their intended use. @@ -80,11 +82,11 @@ Measurements are made by recording values and information about the values with an instrument. How these measurements are recorded depends on the instrument. Measurements for synchronous instruments ([Int64Counter], [Int64UpDownCounter], -[Int64Histogram], [Float64Counter], [Float64UpDownCounter], and -[Float64Histogram]) are recorded using the instrument methods directly. All -counter instruments have an Add method that is used to measure an increment -value, and all histogram instruments have a Record method to measure a data -point. +[Int64Histogram], [Int64Gauge], [Float64Counter], [Float64UpDownCounter], +[Float64Histogram], and [Float64Gauge]) are recorded using the instrument +methods directly. All counter instruments have an Add method that is used to +measure an increment value, and all histogram and synchronous gauge +instruments have a Record method to measure a data point. Asynchronous instruments ([Int64ObservableCounter], [Int64ObservableUpDownCounter], [Int64ObservableGauge], @@ -107,6 +109,31 @@ respectively): If the criteria are not met, use the RegisterCallback method of the [Meter] that created the instrument to register a [Callback]. +# Avoiding Expensive Computations + +All synchronous instruments provide an Enabled method that reports whether the +instrument will process measurements for the given context. When no SDK is +registered or the instrument is otherwise disabled, Enabled returns false. This +can be used to avoid expensive measurement work when a measurement will not be +recorded: + + if counter.Enabled(ctx) { + counter.Add(ctx, 1, metric.WithAttributes(expensiveAttributes()...)) + } + +This is especially valuable when computing attributes is expensive. +[WithAttributes] performs non-trivial work on every call to build an +[attribute.Set] from the provided attributes, and that work is wasted if the +measurement is not recorded. + +For performance sensitive code where the same attribute set is used repeatedly, +prefer [WithAttributeSet]. It accepts a pre-built [attribute.Set], letting you +pay the construction cost once and reuse it across many measurements: + + attrs := attribute.NewSet(attribute.String("key", "val")) + // ... later, on each call: + counter.Add(ctx, 1, metric.WithAttributeSet(attrs)) + # API Implementations This package does not conform to the standard Go versioning policy, all of its diff --git a/vendor/go.opentelemetry.io/otel/metric/instrument.go b/vendor/go.opentelemetry.io/otel/metric/instrument.go index 9f48d5f11..2e79ab568 100644 --- a/vendor/go.opentelemetry.io/otel/metric/instrument.go +++ b/vendor/go.opentelemetry.io/otel/metric/instrument.go @@ -3,7 +3,9 @@ package metric // import "go.opentelemetry.io/otel/metric" -import "go.opentelemetry.io/otel/attribute" +import ( + "go.opentelemetry.io/otel/attribute" +) // Observable is used as a grouping mechanism for all instruments that are // updated within a Callback. @@ -228,6 +230,9 @@ type AddConfig struct { func NewAddConfig(opts []AddOption) AddConfig { config := AddConfig{attrs: *attribute.EmptySet()} for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyAdd(config) } return config @@ -253,6 +258,9 @@ type RecordConfig struct { func NewRecordConfig(opts []RecordOption) RecordConfig { config := RecordConfig{attrs: *attribute.EmptySet()} for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyRecord(config) } return config @@ -278,6 +286,9 @@ type ObserveConfig struct { func NewObserveConfig(opts []ObserveOption) ObserveConfig { config := ObserveConfig{attrs: *attribute.EmptySet()} for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyObserve(config) } return config @@ -299,6 +310,10 @@ type attrOpt struct { set attribute.Set } +func (o *attrOpt) Set(set attribute.Set) { + o.set = set +} + // mergeSets returns the union of keys between a and b. Any duplicate keys will // use the value associated with b. func mergeSets(a, b attribute.Set) attribute.Set { @@ -311,7 +326,7 @@ func mergeSets(a, b attribute.Set) attribute.Set { return attribute.NewSet(merged...) } -func (o attrOpt) applyAdd(c AddConfig) AddConfig { +func (o *attrOpt) applyAdd(c AddConfig) AddConfig { switch { case o.set.Len() == 0: case c.attrs.Len() == 0: @@ -322,7 +337,7 @@ func (o attrOpt) applyAdd(c AddConfig) AddConfig { return c } -func (o attrOpt) applyRecord(c RecordConfig) RecordConfig { +func (o *attrOpt) applyRecord(c RecordConfig) RecordConfig { switch { case o.set.Len() == 0: case c.attrs.Len() == 0: @@ -333,7 +348,7 @@ func (o attrOpt) applyRecord(c RecordConfig) RecordConfig { return c } -func (o attrOpt) applyObserve(c ObserveConfig) ObserveConfig { +func (o *attrOpt) applyObserve(c ObserveConfig) ObserveConfig { switch { case o.set.Len() == 0: case c.attrs.Len() == 0: @@ -350,8 +365,14 @@ func (o attrOpt) applyObserve(c ObserveConfig) ObserveConfig { // If multiple WithAttributeSet or WithAttributes options are passed the // attributes will be merged together in the order they are passed. Attributes // with duplicate keys will use the last value passed. +// +// Experimental: The returned option may implement +// [go.opentelemetry.io/otel/metric/x.Settable][attribute.Set], which can be +// used to replace the option's attribute set and reuse the option without +// additional allocations. This behavior is experimental and may be changed or +// removed in a future release without notice. func WithAttributeSet(attributes attribute.Set) MeasurementOption { - return attrOpt{set: attributes} + return &attrOpt{set: attributes} } // WithAttributes converts attributes into an attribute Set and sets the Set to @@ -369,8 +390,14 @@ func WithAttributeSet(attributes attribute.Set) MeasurementOption { // // See [WithAttributeSet] for information about how multiple WithAttributes are // merged. +// +// Experimental: The returned option may implement +// [go.opentelemetry.io/otel/metric/x.Settable][[]attribute.KeyValue], which can be +// used to replace the option's attributes and reuse the option without +// additional allocations. This behavior is experimental and may be changed or +// removed in a future release without notice. func WithAttributes(attributes ...attribute.KeyValue) MeasurementOption { cp := make([]attribute.KeyValue, len(attributes)) copy(cp, attributes) - return attrOpt{set: attribute.NewSet(cp...)} + return &attrOpt{set: attribute.NewSet(cp...)} } diff --git a/vendor/go.opentelemetry.io/otel/metric/syncfloat64.go b/vendor/go.opentelemetry.io/otel/metric/syncfloat64.go index abb3051d7..2101f686a 100644 --- a/vendor/go.opentelemetry.io/otel/metric/syncfloat64.go +++ b/vendor/go.opentelemetry.io/otel/metric/syncfloat64.go @@ -51,6 +51,9 @@ type Float64CounterConfig struct { func NewFloat64CounterConfig(opts ...Float64CounterOption) Float64CounterConfig { var config Float64CounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64Counter(config) } return config @@ -116,6 +119,9 @@ type Float64UpDownCounterConfig struct { func NewFloat64UpDownCounterConfig(opts ...Float64UpDownCounterOption) Float64UpDownCounterConfig { var config Float64UpDownCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64UpDownCounter(config) } return config @@ -182,6 +188,9 @@ type Float64HistogramConfig struct { func NewFloat64HistogramConfig(opts ...Float64HistogramOption) Float64HistogramConfig { var config Float64HistogramConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64Histogram(config) } return config @@ -251,6 +260,9 @@ type Float64GaugeConfig struct { func NewFloat64GaugeConfig(opts ...Float64GaugeOption) Float64GaugeConfig { var config Float64GaugeConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyFloat64Gauge(config) } return config diff --git a/vendor/go.opentelemetry.io/otel/metric/syncint64.go b/vendor/go.opentelemetry.io/otel/metric/syncint64.go index 5bbfaf039..425c1a0d5 100644 --- a/vendor/go.opentelemetry.io/otel/metric/syncint64.go +++ b/vendor/go.opentelemetry.io/otel/metric/syncint64.go @@ -51,6 +51,9 @@ type Int64CounterConfig struct { func NewInt64CounterConfig(opts ...Int64CounterOption) Int64CounterConfig { var config Int64CounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64Counter(config) } return config @@ -116,6 +119,9 @@ type Int64UpDownCounterConfig struct { func NewInt64UpDownCounterConfig(opts ...Int64UpDownCounterOption) Int64UpDownCounterConfig { var config Int64UpDownCounterConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64UpDownCounter(config) } return config @@ -182,6 +188,9 @@ type Int64HistogramConfig struct { func NewInt64HistogramConfig(opts ...Int64HistogramOption) Int64HistogramConfig { var config Int64HistogramConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64Histogram(config) } return config @@ -251,6 +260,9 @@ type Int64GaugeConfig struct { func NewInt64GaugeConfig(opts ...Int64GaugeOption) Int64GaugeConfig { var config Int64GaugeConfig for _, o := range opts { + if _, ok := o.(experimentalOption); ok { + continue + } config = o.applyInt64Gauge(config) } return config diff --git a/vendor/go.opentelemetry.io/otel/propagation/baggage.go b/vendor/go.opentelemetry.io/otel/propagation/baggage.go index 2ecca3fed..d81b709a2 100644 --- a/vendor/go.opentelemetry.io/otel/propagation/baggage.go +++ b/vendor/go.opentelemetry.io/otel/propagation/baggage.go @@ -5,6 +5,9 @@ package propagation // import "go.opentelemetry.io/otel/propagation" import ( "context" + "errors" + "fmt" + "sync" "go.opentelemetry.io/otel/baggage" "go.opentelemetry.io/otel/internal/errorhandler" @@ -13,11 +16,18 @@ import ( const ( baggageHeader = "baggage" + maxParseErrors = 5 + // W3C Baggage specification limits. // https://www.w3.org/TR/baggage/#limits - maxMembers = 64 + maxMembers = 64 + maxBytesPerBaggageString = 8192 ) +// handleExtractErrOnce limits error reporting for attacker-controlled baggage headers +// to one process-wide emission, preventing repeated extraction from flooding logs. +var handleExtractErrOnce sync.Once + // Baggage is a propagator that supports the W3C Baggage format. // // This propagates user-defined baggage associated with a trace. The complete @@ -57,7 +67,9 @@ func extractSingleBaggage(parent context.Context, carrier TextMapCarrier) contex bag, err := baggage.Parse(bStr) if err != nil { - errorhandler.GetErrorHandler().Handle(err) + handleExtractErrOnce.Do(func() { + errorhandler.GetErrorHandler().Handle(err) + }) } if bag.Len() == 0 { return parent @@ -72,24 +84,60 @@ func extractMultiBaggage(parent context.Context, carrier ValuesGetter) context.C } var members []baggage.Member - for _, bStr := range bVals { - currBag, err := baggage.Parse(bStr) - if err != nil { - errorhandler.GetErrorHandler().Handle(err) + var totalBytes int + var parseErrors int + var truncateErr error + for i, bStr := range bVals { + if i > 0 { + totalBytes++ // comma separator between combined header values } - if currBag.Len() == 0 { - continue + totalBytes += len(bStr) + if totalBytes > maxBytesPerBaggageString { + // Per the W3C Baggage spec, the byte limit applies to the + // combination of all baggage headers, not each header + // individually. Mirror the single-header behavior of + // reporting the error and returning the parent context + // with no baggage attached. + handleExtractErrOnce.Do(func() { + errorhandler.GetErrorHandler().Handle(fmt.Errorf( + "baggage: aggregate header size %d exceeds %d byte limit", + totalBytes, + maxBytesPerBaggageString, + )) + }) + return parent } - members = append(members, currBag.Members()...) - if len(members) >= maxMembers { - break + + // If members exceed the limit, stop parsing baggage. + if len(members) <= maxMembers { + currBag, err := baggage.Parse(bStr) + if err != nil { + parseErrors++ + if parseErrors <= maxParseErrors { + truncateErr = errors.Join(truncateErr, err) + } + } + if currBag.Len() == 0 { + continue + } + members = append(members, currBag.Members()...) } } + if dropped := parseErrors - maxParseErrors; dropped > 0 { + truncateErr = errors.Join(truncateErr, fmt.Errorf("and %d more error(s)", dropped)) + } + b, err := baggage.New(members...) if err != nil { - errorhandler.GetErrorHandler().Handle(err) + truncateErr = errors.Join(truncateErr, err) } + if truncateErr != nil { + handleExtractErrOnce.Do(func() { + errorhandler.GetErrorHandler().Handle(truncateErr) + }) + } + if b.Len() == 0 { return parent } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/config.go b/vendor/go.opentelemetry.io/otel/sdk/metric/config.go index 306e5e3cd..dda4e086d 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/config.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/config.go @@ -25,7 +25,7 @@ type config struct { cardinalityLimit int } -const defaultCardinalityLimit = 0 +const defaultCardinalityLimit = 2000 // readerSignals returns a force-flush and shutdown function for a // MeterProvider to call in their respective options. All Readers c contains @@ -70,6 +70,10 @@ func unifyShutdown(funcs []func(context.Context) error) func(context.Context) er } } +type experimentalOption interface { + Experimental() +} + // newConfig returns a config configured with options. func newConfig(options []Option) config { conf := config{ @@ -81,6 +85,9 @@ func newConfig(options []Option) config { conf = o.apply(conf) } for _, o := range options { + if _, ok := o.(experimentalOption); ok { + continue + } conf = o.apply(conf) } return conf @@ -150,7 +157,7 @@ func WithView(views ...View) Option { // exemplar reservoir, but the exemplar reservoir makes the final decision of // whether to store an exemplar. // -// By default, the [exemplar.SampledFilter] +// By default, the [exemplar.TraceBasedFilter] // is used. Exemplars can be entirely disabled by providing the // [exemplar.AlwaysOffFilter]. func WithExemplarFilter(filter exemplar.Filter) Option { @@ -165,7 +172,10 @@ func WithExemplarFilter(filter exemplar.Filter) Option { // The cardinality limit is the hard limit on the number of metric datapoints // that can be collected for a single instrument in a single collect cycle. // -// Setting this to a zero or negative value means no limit is applied. +// By default, if this option is not used, a limit of +// 2000 is applied. +// +// Setting this to a zero or negative means no limit is applied. // This value applies to all instrument kinds, but can be overridden per kind by // the reader's cardinality limit selector (see [WithCardinalityLimitSelector]). func WithCardinalityLimit(limit int) Option { diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/doc.go b/vendor/go.opentelemetry.io/otel/sdk/metric/doc.go index dd75eefac..51e168c75 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/doc.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/doc.go @@ -24,6 +24,10 @@ // View. Views allow users that run OpenTelemetry instrumented code to modify // the generated data of that instrumentation. // +// Note that attributes filtered out by a View may still appear on Exemplars, +// because Exemplars are recorded with the dropped measurement attributes +// when View attribute filtering is applied. +// // The data generated by a MeterProvider needs to include information about its // origin. A MeterProvider needs to be configured with a Resource, using the // WithResource MeterProviderOption, to include this information. This Resource @@ -44,9 +48,7 @@ // Cardinality refers to the number of unique attributes collected. High cardinality can lead to // excessive memory usage, increased storage costs, and backend performance issues. // -// Currently, the OpenTelemetry Go Metric SDK does not enforce a cardinality limit by default -// (note that this may change in a future release). Use [WithCardinalityLimit] to set the -// cardinality limit as desired. +// By default, the OpenTelemetry Go Metric SDK enforces a cardinality limit of 2000. // // New attribute sets are dropped when the cardinality limit is reached. The measurement of // these sets are aggregated into @@ -57,8 +59,8 @@ // // Recommendations: // -// - Set the limit based on the theoretical maximum combinations or expected -// active combinations. The OpenTelemetry Specification recommends a default of 2000. +// - Tune the limit based on the theoretical maximum combinations or expected +// active combinations. The SDK default is 2000. // - A too high of a limit increases worst-case memory overhead in the SDK and may cause downstream // issues for databases that cannot handle high cardinality. // - A too low of a limit causes loss of attribute detail as more data falls into overflow. diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar.go b/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar.go index 38b8745e6..993f001ae 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar.go @@ -4,6 +4,7 @@ package metric // import "go.opentelemetry.io/otel/sdk/metric" import ( + "reflect" "runtime" "go.opentelemetry.io/otel/attribute" @@ -19,9 +20,19 @@ type ExemplarReservoirProviderSelector func(Aggregation) exemplar.ReservoirProvi // reservoirFunc returns the appropriately configured exemplar reservoir // creation func based on the passed InstrumentKind and filter configuration. func reservoirFunc[N int64 | float64]( + kind InstrumentKind, provider exemplar.ReservoirProvider, filter exemplar.Filter, ) func(attribute.Set) aggregate.FilteredExemplarReservoir[N] { + if reflect.ValueOf(filter).Pointer() == reflect.ValueOf(exemplar.AlwaysOffFilter).Pointer() { + return aggregate.DropReservoir[N] + } + if (kind == InstrumentKindObservableCounter || kind == InstrumentKindObservableUpDownCounter || kind == InstrumentKindObservableGauge) && + reflect.ValueOf(filter).Pointer() == reflect.ValueOf(exemplar.TraceBasedFilter).Pointer() { + // Asynchronous instruments do not accept context, so TraceBasedFilter + // will never record any exemplars. + return aggregate.DropReservoir[N] + } return func(attrs attribute.Set) aggregate.FilteredExemplarReservoir[N] { return aggregate.NewFilteredExemplarReservoir[N](filter, provider(attrs)) } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar/fixed_size_reservoir.go b/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar/fixed_size_reservoir.go index 15d713899..ce97ab1bf 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar/fixed_size_reservoir.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/exemplar/fixed_size_reservoir.go @@ -8,7 +8,6 @@ import ( "math" "math/rand/v2" "sync" - "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" @@ -27,19 +26,7 @@ func FixedSizeReservoirProvider(k int) ReservoirProvider { // sample each one. If there are more than k, the Reservoir will then randomly // sample all additional measurement with a decreasing probability. func NewFixedSizeReservoir(k int) *FixedSizeReservoir { - if k < 0 { - k = 0 - } - // Use math.MaxInt32 instead of math.MaxUint32 to prevent overflowing int - // on 32-bit systems. - if k > math.MaxInt32 { - k = math.MaxInt32 - } - return &FixedSizeReservoir{ - storage: newStorage(k), - // Above we ensure k is positive, and less than MaxInt32. - nextTracker: newNextTracker(uint32(k)), // nolint: gosec - } + return newFixedSizeReservoir(newStorage(k)) } var _ Reservoir = &FixedSizeReservoir{} @@ -51,7 +38,42 @@ var _ Reservoir = &FixedSizeReservoir{} type FixedSizeReservoir struct { reservoir.ConcurrentSafe *storage - *nextTracker + mu sync.Mutex + + // count is the number of measurement seen. + count int64 + // next is the next count that will store a measurement at a random index + // once the reservoir has been filled. + next int64 + // w is the largest random number in a distribution that is used to compute + // the next next. + w float64 +} + +func newFixedSizeReservoir(s *storage) *FixedSizeReservoir { + r := &FixedSizeReservoir{ + storage: s, + } + if cap(r.measurements) > 0 { + r.reset() + } + return r +} + +// randomFloat64 returns, as a float64, a uniform pseudo-random number in the +// open interval (0.0,1.0). +func (*FixedSizeReservoir) randomFloat64() float64 { + // TODO: Use an algorithm that avoids rejection sampling. For example: + // + // const precision = 1 << 53 // 2^53 + // // Generate an integer in [1, 2^53 - 1] + // v := rand.Uint64() % (precision - 1) + 1 + // return float64(v) / float64(precision) + f := rand.Float64() + for f == 0 { + f = rand.Float64() + } + return f } // Offer accepts the parameters associated with a measurement. The @@ -66,6 +88,10 @@ type FixedSizeReservoir struct { // parameters are the value and dropped (filtered) attributes of the // measurement respectively. func (r *FixedSizeReservoir) Offer(ctx context.Context, t time.Time, n Value, a []attribute.KeyValue) { + if cap(r.measurements) == 0 { + return + } + // The following algorithm is "Algorithm L" from Li, Kim-Hung (4 December // 1994). "Reservoir-Sampling Algorithms of Time Complexity // O(n(1+log(N/n)))". ACM Transactions on Mathematical Software. 20 (4): @@ -107,65 +133,25 @@ func (r *FixedSizeReservoir) Offer(ctx context.Context, t time.Time, n Value, a // https://github.com/MrAlias/reservoir-sampling for a performance // comparison of reservoir sampling algorithms. - count, next := r.incrementCount() - if count < r.k { - r.store(ctx, int(count), t, n, a) - } else if count == next { + r.mu.Lock() + defer r.mu.Unlock() + if int(r.count) < cap(r.measurements) { + r.store(ctx, int(r.count), t, n, a) + } else if r.count == r.next { // Overwrite a random existing measurement with the one offered. - idx := rand.IntN(int(r.k)) + idx := int(rand.Int64N(int64(cap(r.measurements)))) r.store(ctx, idx, t, n, a) - r.wMu.Lock() - defer r.wMu.Unlock() - newCount, newNext := r.loadCountAndNext() - if newNext < next || newCount < count { - // This Observe() raced with Collect(), and r.reset() has been - // called since r.incrementCount(). Skip the call to advance in - // this case because our exemplar may have been collected in the - // previous interval. - return - } r.advance() } -} - -// Collect returns all the held exemplars. -// -// The Reservoir state is preserved after this call. -func (r *FixedSizeReservoir) Collect(dest *[]Exemplar) { - r.storage.Collect(dest) - // Call reset here even though it will reset r.count and restart the random - // number series. This will persist any old exemplars as long as no new - // measurements are offered, but it will also prioritize those new - // measurements that are made over the older collection cycle ones. - r.reset() -} - -func newNextTracker(k uint32) *nextTracker { - nt := &nextTracker{k: k} - nt.reset() - return nt -} - -type nextTracker struct { - // countAndNext holds the current counts in the lower 32 bits and the next - // value in the upper 32 bits. - countAndNext atomic.Uint64 - // w is the largest random number in a distribution that is used to compute - // the next next. - w float64 - // wMu ensures w is kept consistent with next during advance and reset. - wMu sync.Mutex - // k is the number of measurements that can be stored in the reservoir. - k uint32 + r.count++ } // reset resets r to the initial state. -func (r *nextTracker) reset() { - r.wMu.Lock() - defer r.wMu.Unlock() +func (r *FixedSizeReservoir) reset() { // This resets the number of exemplars known. + r.count = 0 // Random index inserts should only happen after the storage is full. - r.setCountAndNext(0, r.k) + r.next = int64(cap(r.measurements)) // Initial random number in the series used to generate r.next. // @@ -176,40 +162,14 @@ func (r *nextTracker) reset() { // This maps the uniform random number in (0,1) to a geometric distribution // over the same interval. The mean of the distribution is inversely // proportional to the storage capacity. - r.w = math.Exp(math.Log(randomFloat64()) / float64(r.k)) + r.w = math.Exp(math.Log(r.randomFloat64()) / float64(cap(r.measurements))) r.advance() } -// incrementCount increments the count. It returns the count before the -// increment and the current next value. -func (r *nextTracker) incrementCount() (uint32, uint32) { - n := r.countAndNext.Add(1) - // Both count and next are stored in the upper and lower 32 bits, and thus - // can't overflow. - return uint32(n&((1<<32)-1) - 1), uint32(n >> 32) // nolint: gosec -} - -// incrementNext increments the next value. -func (r *nextTracker) incrementNext(inc uint32) { - r.countAndNext.Add(uint64(inc) << 32) -} - -// setCountAndNext sets the count and next values. -func (r *nextTracker) setCountAndNext(count, next uint32) { - r.countAndNext.Store(uint64(next)<<32 + uint64(count)) -} - -func (r *nextTracker) loadCountAndNext() (uint32, uint32) { - n := r.countAndNext.Load() - // Both count and next are stored in the upper and lower 32 bits, and thus - // can't overflow. - return uint32(n&((1<<32)-1) - 1), uint32(n >> 32) // nolint: gosec -} - // advance updates the count at which the offered measurement will overwrite an // existing exemplar. -func (r *nextTracker) advance() { +func (r *FixedSizeReservoir) advance() { // Calculate the next value in the random number series. // // The current value of r.w is based on the max of a distribution of random @@ -222,7 +182,7 @@ func (r *nextTracker) advance() { // therefore the next r.w will be based on the same distribution (i.e. // `max(u_1,u_2,...,u_k)`). Therefore, we can sample the next r.w by // computing the next random number `u` and take r.w as `w * u^(1/k)`. - r.w *= math.Exp(math.Log(randomFloat64()) / float64(r.k)) + r.w *= math.Exp(math.Log(r.randomFloat64()) / float64(cap(r.measurements))) // Use the new random number in the series to calculate the count of the // next measurement that will be stored. // @@ -233,21 +193,23 @@ func (r *nextTracker) advance() { // // Important to note, the new r.next will always be at least 1 more than // the last r.next. - r.incrementNext(uint32(math.Log(randomFloat64())/math.Log(1-r.w)) + 1) + r.next += int64(math.Log(r.randomFloat64())/math.Log(1-r.w)) + 1 } -// randomFloat64 returns, as a float64, a uniform pseudo-random number in the -// open interval (0.0,1.0). -func randomFloat64() float64 { - // TODO: Use an algorithm that avoids rejection sampling. For example: - // - // const precision = 1 << 53 // 2^53 - // // Generate an integer in [1, 2^53 - 1] - // v := rand.Uint64() % (precision - 1) + 1 - // return float64(v) / float64(precision) - f := rand.Float64() - for f == 0 { - f = rand.Float64() +// Collect returns all the held exemplars. +// +// The Reservoir state is preserved after this call. +func (r *FixedSizeReservoir) Collect(dest *[]Exemplar) { + if cap(r.measurements) == 0 { + *dest = (*dest)[:0] + return } - return f + r.mu.Lock() + defer r.mu.Unlock() + r.storage.Collect(dest) + // Call reset here even though it will reset r.count and restart the random + // number series. This will persist any old exemplars as long as no new + // measurements are offered, but it will also prioritize those new + // measurements that are made over the older collection cycle ones. + r.reset() } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/instrument.go b/vendor/go.opentelemetry.io/otel/sdk/metric/instrument.go index b08052559..dfb4964d9 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/instrument.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/instrument.go @@ -141,6 +141,10 @@ type Stream struct { // the attribute will not be recorded, otherwise, if it returns true, it // will record the attribute. // + // Note that attributes filtered out by a View may still appear on Exemplars, + // because Exemplars are recorded with the dropped measurement attributes + // when View attribute filtering is applied. + // // Use NewAllowKeysFilter from "go.opentelemetry.io/otel/attribute" to // provide an allow-list of attribute keys here. AttributeFilter attribute.Filter diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/aggregate.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/aggregate.go index a1ae55737..a35f3bcc1 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/aggregate.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/aggregate.go @@ -36,8 +36,8 @@ type Builder[N int64 | float64] struct { // ReservoirFunc is the factory function used by aggregate functions to // create new exemplar reservoirs for a new seen attribute set. // - // If this is not provided a default factory function that returns an - // dropReservoir reservoir will be used. + // If this is not provided a default factory function that returns a + // DropReservoir reservoir will be used. ReservoirFunc func(attribute.Set) FilteredExemplarReservoir[N] // AggregationLimit is the cardinality limit of measurement attributes. Any // measurement for new attributes once the limit has been reached will be @@ -54,7 +54,7 @@ func (b Builder[N]) resFunc() func(attribute.Set) FilteredExemplarReservoir[N] { return b.ReservoirFunc } - return dropReservoir + return DropReservoir } type fltrMeasure[N int64 | float64] func(ctx context.Context, value N, fltrAttr attribute.Set, droppedAttr []attribute.KeyValue) diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/drop.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/drop.go index 129920cbd..7adfcaf85 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/drop.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/drop.go @@ -10,8 +10,8 @@ import ( "go.opentelemetry.io/otel/sdk/metric/exemplar" ) -// dropReservoir returns a [FilteredReservoir] that drops all measurements it is offered. -func dropReservoir[N int64 | float64](attribute.Set) FilteredExemplarReservoir[N] { +// DropReservoir returns a [FilteredExemplarReservoir] that drops all measurements it is offered. +func DropReservoir[N int64 | float64](attribute.Set) FilteredExemplarReservoir[N] { return &dropRes[N]{} } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/exponential_histogram.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/exponential_histogram.go index 312d73c45..767b1d6dc 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/exponential_histogram.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/exponential_histogram.go @@ -26,8 +26,9 @@ const ( // expoHistogramDataPoint is a single data point in an exponential histogram. type expoHistogramDataPoint[N int64 | float64] struct { - attrs attribute.Set - res FilteredExemplarReservoir[N] + attrs attribute.Set + res FilteredExemplarReservoir[N] + dropExemplars bool minMax atomicMinMax[N] sum atomicCounter[N] @@ -349,13 +350,18 @@ func (e *expoHistogram[N]) measure( v, ok = e.values[fltrAttr.Equivalent()] if !ok { v = newExpoHistogramDataPoint[N](fltrAttr, e.maxSize, e.maxScale, e.noMinMax, e.noSum) - v.res = e.newRes(fltrAttr) + r := e.newRes(fltrAttr) + _, isDrop := r.(*dropRes[N]) + v.res = r + v.dropExemplars = isDrop e.values[fltrAttr.Equivalent()] = v } } v.record(value) - v.res.Offer(ctx, value, droppedAttr) + if !v.dropExemplars { + v.res.Offer(ctx, value, droppedAttr) + } } func (e *expoHistogram[N]) delta( diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/histogram.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/histogram.go index 83582c670..50c82ff32 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/histogram.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/histogram.go @@ -17,8 +17,9 @@ import ( // histogramPoint is a single histogram point, used in delta aggregations. type histogramPoint[N int64 | float64] struct { - attrs attribute.Set - res FilteredExemplarReservoir[N] + attrs attribute.Set + res FilteredExemplarReservoir[N] + dropExemplars bool histogramPointCounters[N] } @@ -28,9 +29,10 @@ type hotColdHistogramPoint[N int64 | float64] struct { hcwg hotColdWaitGroup hotColdPoint [2]histogramPointCounters[N] - attrs attribute.Set - res FilteredExemplarReservoir[N] - startTime time.Time + attrs attribute.Set + res FilteredExemplarReservoir[N] + startTime time.Time + dropExemplars bool } // histogramPointCounters contains only the atomic counter data, and is used by @@ -113,9 +115,12 @@ func (s *deltaHistogram[N]) measure( hotIdx := s.hcwg.start() defer s.hcwg.done(hotIdx) h := s.hotColdValMap[hotIdx].LoadOrStoreAttr(fltrAttr, func(attr attribute.Set) any { + r := s.newRes(attr) + _, isDrop := r.(*dropRes[N]) hPt := &histogramPoint[N]{ - res: s.newRes(attr), - attrs: attr, + res: r, + attrs: attr, + dropExemplars: isDrop, // N+1 buckets. For example: // // bounds = [0, 5, 10] @@ -141,7 +146,9 @@ func (s *deltaHistogram[N]) measure( if !s.noSum { h.total.add(value) } - h.res.Offer(ctx, value, droppedAttr) + if !h.dropExemplars { + h.res.Offer(ctx, value, droppedAttr) + } } // newDeltaHistogram returns a histogram that is reset each time it is @@ -282,9 +289,13 @@ func (s *cumulativeHistogram[N]) measure( droppedAttr []attribute.KeyValue, ) { h := s.values.LoadOrStoreAttr(fltrAttr, func(attr attribute.Set) any { + r := s.newRes(attr) + _, isDrop := r.(*dropRes[N]) hPt := &hotColdHistogramPoint[N]{ - res: s.newRes(attr), - attrs: attr, + res: r, + attrs: attr, + startTime: now(), + dropExemplars: isDrop, // N+1 buckets. For example: // // bounds = [0, 5, 10] @@ -300,7 +311,6 @@ func (s *cumulativeHistogram[N]) measure( counts: make([]atomic.Uint64, len(s.bounds)+1), }, }, - startTime: now(), } return hPt }).(*hotColdHistogramPoint[N]) @@ -322,7 +332,9 @@ func (s *cumulativeHistogram[N]) measure( if !s.noSum { h.hotColdPoint[hotIdx].total.add(value) } - h.res.Offer(ctx, value, droppedAttr) + if !h.dropExemplars { + h.res.Offer(ctx, value, droppedAttr) + } } func (s *cumulativeHistogram[N]) collect( diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/lastvalue.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/lastvalue.go index 4c004bc99..2af252ec5 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/lastvalue.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/lastvalue.go @@ -14,10 +14,11 @@ import ( // lastValuePoint is timestamped measurement data. type lastValuePoint[N int64 | float64] struct { - attrs attribute.Set - value atomicN[N] - res FilteredExemplarReservoir[N] - startTime time.Time + attrs attribute.Set + value atomicN[N] + res FilteredExemplarReservoir[N] + startTime time.Time + dropExemplars bool } // lastValueMap summarizes a set of measurements as the last one made. @@ -33,17 +34,22 @@ func (s *lastValueMap[N]) measure( droppedAttr []attribute.KeyValue, ) { lv := s.values.LoadOrStoreAttr(fltrAttr, func(attr attribute.Set) any { + r := s.newRes(attr) + _, isDrop := r.(*dropRes[N]) p := &lastValuePoint[N]{ - res: s.newRes(attr), - attrs: attr, - startTime: now(), + res: r, + attrs: attr, + startTime: now(), + dropExemplars: isDrop, } p.value.Store(value) return p }).(*lastValuePoint[N]) lv.value.Store(value) - lv.res.Offer(ctx, value, droppedAttr) + if !lv.dropExemplars { + lv.res.Offer(ctx, value, droppedAttr) + } } func newDeltaLastValue[N int64 | float64]( diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/sum.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/sum.go index 3fe7c7cf0..0cd3c9457 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/sum.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/aggregate/sum.go @@ -13,10 +13,11 @@ import ( ) type sumValue[N int64 | float64] struct { - n atomicCounter[N] - res FilteredExemplarReservoir[N] - attrs attribute.Set - startTime time.Time + n atomicCounter[N] + res FilteredExemplarReservoir[N] + attrs attribute.Set + startTime time.Time + dropExemplars bool } type sumValueMap[N int64 | float64] struct { @@ -31,17 +32,22 @@ func (s *sumValueMap[N]) measure( droppedAttr []attribute.KeyValue, ) { sv := s.values.LoadOrStoreAttr(fltrAttr, func(attr attribute.Set) any { + r := s.newRes(attr) + _, isDrop := r.(*dropRes[N]) return &sumValue[N]{ - res: s.newRes(attr), - attrs: attr, - startTime: now(), + res: r, + attrs: attr, + startTime: now(), + dropExemplars: isDrop, } }).(*sumValue[N]) sv.n.add(value) // It is possible for collection to race with measurement and observe the // exemplar in the batch of metrics after the add() for cumulative sums. // This is an accepted tradeoff to avoid locking during measurement. - sv.res.Offer(ctx, value, droppedAttr) + if !sv.dropExemplars { + sv.res.Offer(ctx, value, droppedAttr) + } } // newDeltaSum returns an aggregator that summarizes a set of measurements as diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/observ/instrumentation.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/observ/instrumentation.go index 2d2b987c5..c580dcc30 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/observ/instrumentation.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/observ/instrumentation.go @@ -16,8 +16,8 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/internal/x" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( @@ -54,6 +54,7 @@ var ( func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } func put[T any](p *sync.Pool, s *[]T) { + clear(*s) *s = (*s)[:0] // Reset. p.Put(s) } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/README.md b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/README.md new file mode 100644 index 000000000..41a4c2996 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/README.md @@ -0,0 +1,42 @@ +# Experimental Features + +The Metric SDK contains features that have not yet stabilized in the OpenTelemetry specification. +These features are added to the OpenTelemetry Go Metric SDK prior to stabilization in the specification so that users can start experimenting with them and provide feedback. + +These feature may change in backwards incompatible ways as feedback is applied. +See the [Compatibility and Stability](#compatibility-and-stability) section for more information. + +## Features + +- [Metric Export Batch Size](#metric-export-batch-size) + +### Metric Export Batch Size + +The metric export can be split into batches before exporting by specifying a maximum number of data points per batch. + +This experimental feature can be enabled by setting the `OTEL_GO_X_METRIC_EXPORT_BATCH_SIZE` environment variable. +The value MUST be a positive integer. +All other values or an empty value will result in the default behavior of not batching. + +#### Examples + +Enable metrics to be batched by maximum export batch size of 200. + +```console +export OTEL_GO_X_METRIC_EXPORT_BATCH_SIZE=200 +``` + +Disable metric export batching. + +```console +unset OTEL_GO_X_METRIC_EXPORT_BATCH_SIZE +``` + +## Compatibility and Stability + +Experimental features do not fall within the scope of the OpenTelemetry Go versioning and stability [policy](../../../../VERSIONING.md). +These features may be removed or modified in successive version releases, including patch versions. + +When an experimental feature is promoted to a stable feature, a migration path will be included in the changelog entry of the release. +There is no guarantee that any environment variable feature flags that enabled the experimental feature will be supported by the stable version. +If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/x.go b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/x.go new file mode 100644 index 000000000..8e3c62a13 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/internal/x/x.go @@ -0,0 +1,70 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package x contains support for OTel metric SDK experimental features. +// +// This package should only be used for features defined in the specification. +// It should not be used for experiments or new project ideas. +package x // import "go.opentelemetry.io/otel/sdk/metric/internal/x" + +import ( + "os" + "strconv" +) + +// Feature is an experimental feature control flag. It provides a uniform way +// to interact with these feature flags and parse their values. +type Feature[T any] struct { + key string + parse func(v string) (T, bool) +} + +//nolint:unused +func newFeature[T any](suffix string, parse func(string) (T, bool)) Feature[T] { + const envKeyRoot = "OTEL_GO_X_" + return Feature[T]{ + key: envKeyRoot + suffix, + parse: parse, + } +} + +// Key returns the environment variable key that needs to be set to enable the +// feature. +func (f Feature[T]) Key() string { return f.key } + +// Lookup returns the user configured value for the feature and true if the +// user has enabled the feature. Otherwise, if the feature is not enabled, a +// zero-value and false are returned. +func (f Feature[T]) Lookup() (v T, ok bool) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/62effed618589a0bec416a87e559c0a9d96289bb/specification/configuration/sdk-environment-variables.md#parsing-empty-value + // + // > The SDK MUST interpret an empty value of an environment variable the + // > same way as when the variable is unset. + vRaw := os.Getenv(f.key) + if vRaw == "" { + return v, ok + } + return f.parse(vRaw) +} + +// Enabled reports whether the feature is enabled. +func (f Feature[T]) Enabled() bool { + _, ok := f.Lookup() + return ok +} + +// MetricExportBatchSize is an experimental feature flag that controls the +// max export batch size for metric data. +// +// To enable this feature set the OTEL_GO_X_METRIC_EXPORT_BATCH_SIZE environment +// variable to a positive integer value. +var MetricExportBatchSize = newFeature( + "METRIC_EXPORT_BATCH_SIZE", + func(v string) (int, bool) { + val, err := strconv.Atoi(v) + if err == nil && val > 0 { + return val, true + } + return 0, false + }, +) diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/manual_reader.go b/vendor/go.opentelemetry.io/otel/sdk/metric/manual_reader.go index 0357afd45..7e0cccdff 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/manual_reader.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/manual_reader.go @@ -169,17 +169,17 @@ func (mr *ManualReader) Collect(ctx context.Context, rm *metricdata.ResourceMetr } // MarshalLog returns logging data about the ManualReader. -func (r *ManualReader) MarshalLog() any { - r.mu.Lock() - down := r.isShutdown - r.mu.Unlock() +func (mr *ManualReader) MarshalLog() any { + mr.mu.Lock() + down := mr.isShutdown + mr.mu.Unlock() return struct { Type string Registered bool Shutdown bool }{ Type: "ManualReader", - Registered: r.sdkProducer.Load() != nil, + Registered: mr.sdkProducer.Load() != nil, Shutdown: down, } } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/meter.go b/vendor/go.opentelemetry.io/otel/sdk/metric/meter.go index e0a1e90e7..e9a2d5957 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/meter.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/meter.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/embedded" @@ -70,7 +71,7 @@ func (m *meter) Int64Counter(name string, options ...metric.Int64CounterOption) cfg := metric.NewInt64CounterConfig(options...) const kind = InstrumentKindCounter p := int64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -88,7 +89,7 @@ func (m *meter) Int64UpDownCounter( cfg := metric.NewInt64UpDownCounterConfig(options...) const kind = InstrumentKindUpDownCounter p := int64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -102,7 +103,7 @@ func (m *meter) Int64UpDownCounter( func (m *meter) Int64Histogram(name string, options ...metric.Int64HistogramOption) (metric.Int64Histogram, error) { cfg := metric.NewInt64HistogramConfig(options...) p := int64InstProvider{m} - i, err := p.lookupHistogram(name, cfg) + i, err := p.lookupHistogram(name, cfg, defaultAttributes(options)) if err != nil { return i, err } @@ -117,7 +118,7 @@ func (m *meter) Int64Gauge(name string, options ...metric.Int64GaugeOption) (met cfg := metric.NewInt64GaugeConfig(options...) const kind = InstrumentKindGauge p := int64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -127,7 +128,11 @@ func (m *meter) Int64Gauge(name string, options ...metric.Int64GaugeOption) (met // int64ObservableInstrument returns a new observable identified by the Instrument. // It registers callbacks for each reader's pipeline. -func (m *meter) int64ObservableInstrument(id Instrument, callbacks []metric.Int64Callback) (int64Observable, error) { +func (m *meter) int64ObservableInstrument( + id Instrument, + allowedKeys []attribute.Key, + callbacks []metric.Int64Callback, +) (int64Observable, error) { key := instID{ Name: id.Name, Description: id.Description, @@ -142,7 +147,7 @@ func (m *meter) int64ObservableInstrument(id Instrument, callbacks []metric.Int6 for _, insert := range m.int64Resolver.inserters { // Connect the measure functions for instruments in this pipeline with the // callbacks for this pipeline. - in, err := insert.Instrument(id, insert.readerDefaultAggregation(id.Kind)) + in, err := insert.Instrument(id, allowedKeys, insert.readerDefaultAggregation(id.Kind)) if err != nil { return inst, err } @@ -188,7 +193,7 @@ func (m *meter) Int64ObservableCounter( Kind: InstrumentKindObservableCounter, Scope: m.scope, } - return m.int64ObservableInstrument(id, cfg.Callbacks()) + return m.int64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } // Int64ObservableUpDownCounter returns a new instrument identified by name and @@ -212,7 +217,7 @@ func (m *meter) Int64ObservableUpDownCounter( Kind: InstrumentKindObservableUpDownCounter, Scope: m.scope, } - return m.int64ObservableInstrument(id, cfg.Callbacks()) + return m.int64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } // Int64ObservableGauge returns a new instrument identified by name and @@ -236,7 +241,7 @@ func (m *meter) Int64ObservableGauge( Kind: InstrumentKindObservableGauge, Scope: m.scope, } - return m.int64ObservableInstrument(id, cfg.Callbacks()) + return m.int64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } // Float64Counter returns a new instrument identified by name and configured @@ -246,7 +251,7 @@ func (m *meter) Float64Counter(name string, options ...metric.Float64CounterOpti cfg := metric.NewFloat64CounterConfig(options...) const kind = InstrumentKindCounter p := float64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -264,7 +269,7 @@ func (m *meter) Float64UpDownCounter( cfg := metric.NewFloat64UpDownCounterConfig(options...) const kind = InstrumentKindUpDownCounter p := float64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -281,7 +286,7 @@ func (m *meter) Float64Histogram( ) (metric.Float64Histogram, error) { cfg := metric.NewFloat64HistogramConfig(options...) p := float64InstProvider{m} - i, err := p.lookupHistogram(name, cfg) + i, err := p.lookupHistogram(name, cfg, defaultAttributes(options)) if err != nil { return i, err } @@ -296,7 +301,7 @@ func (m *meter) Float64Gauge(name string, options ...metric.Float64GaugeOption) cfg := metric.NewFloat64GaugeConfig(options...) const kind = InstrumentKindGauge p := float64InstProvider{m} - i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit()) + i, err := p.lookup(kind, name, cfg.Description(), cfg.Unit(), defaultAttributes(options)) if err != nil { return i, err } @@ -308,6 +313,7 @@ func (m *meter) Float64Gauge(name string, options ...metric.Float64GaugeOption) // It registers callbacks for each reader's pipeline. func (m *meter) float64ObservableInstrument( id Instrument, + allowedKeys []attribute.Key, callbacks []metric.Float64Callback, ) (float64Observable, error) { key := instID{ @@ -316,7 +322,7 @@ func (m *meter) float64ObservableInstrument( Unit: id.Unit, Kind: id.Kind, } - if m.int64ObservableInsts.HasKey(key) && len(callbacks) > 0 { + if m.float64ObservableInsts.HasKey(key) && len(callbacks) > 0 { warnRepeatedObservableCallbacks(id) } return m.float64ObservableInsts.Lookup(key, func() (float64Observable, error) { @@ -324,7 +330,7 @@ func (m *meter) float64ObservableInstrument( for _, insert := range m.float64Resolver.inserters { // Connect the measure functions for instruments in this pipeline with the // callbacks for this pipeline. - in, err := insert.Instrument(id, insert.readerDefaultAggregation(id.Kind)) + in, err := insert.Instrument(id, allowedKeys, insert.readerDefaultAggregation(id.Kind)) if err != nil { return inst, err } @@ -370,7 +376,7 @@ func (m *meter) Float64ObservableCounter( Kind: InstrumentKindObservableCounter, Scope: m.scope, } - return m.float64ObservableInstrument(id, cfg.Callbacks()) + return m.float64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } // Float64ObservableUpDownCounter returns a new instrument identified by name @@ -394,7 +400,7 @@ func (m *meter) Float64ObservableUpDownCounter( Kind: InstrumentKindObservableUpDownCounter, Scope: m.scope, } - return m.float64ObservableInstrument(id, cfg.Callbacks()) + return m.float64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } // Float64ObservableGauge returns a new instrument identified by name and @@ -418,7 +424,7 @@ func (m *meter) Float64ObservableGauge( Kind: InstrumentKindObservableGauge, Scope: m.scope, } - return m.float64ObservableInstrument(id, cfg.Callbacks()) + return m.float64ObservableInstrument(id, defaultAttributes(options), cfg.Callbacks()) } func validateInstrumentName(name string) error { @@ -576,7 +582,8 @@ func (r observer) ObserveFloat64(o metric.Float64Observable, v float64, opts ... if _, registered := r.float64[oImpl.observableID]; !registered { if !oImpl.dropAggregation { - global.Error(errUnregObserver, "failed to record", + global.Error( + errUnregObserver, "failed to record", "name", oImpl.name, "description", oImpl.description, "unit", oImpl.unit, @@ -606,7 +613,8 @@ func (r observer) ObserveInt64(o metric.Int64Observable, v int64, opts ...metric if _, registered := r.int64[oImpl.observableID]; !registered { if !oImpl.dropAggregation { - global.Error(errUnregObserver, "failed to record", + global.Error( + errUnregObserver, "failed to record", "name", oImpl.name, "description", oImpl.description, "unit", oImpl.unit, @@ -633,7 +641,11 @@ func (noopRegister) Unregister() error { // int64InstProvider provides int64 OpenTelemetry instruments. type int64InstProvider struct{ *meter } -func (p int64InstProvider) aggs(kind InstrumentKind, name, desc, u string) ([]aggregate.Measure[int64], error) { +func (p int64InstProvider) aggs( + kind InstrumentKind, + name, desc, u string, + allowedKeys []attribute.Key, +) ([]aggregate.Measure[int64], error) { inst := Instrument{ Name: name, Description: desc, @@ -641,12 +653,13 @@ func (p int64InstProvider) aggs(kind InstrumentKind, name, desc, u string) ([]ag Kind: kind, Scope: p.scope, } - return p.int64Resolver.Aggregators(inst) + return p.int64Resolver.Aggregators(inst, allowedKeys) } func (p int64InstProvider) histogramAggs( name string, cfg metric.Int64HistogramConfig, + allowedKeys []attribute.Key, ) ([]aggregate.Measure[int64], error) { boundaries := cfg.ExplicitBucketBoundaries() aggError := AggregationExplicitBucketHistogram{Boundaries: boundaries}.err() @@ -661,32 +674,40 @@ func (p int64InstProvider) histogramAggs( Kind: InstrumentKindHistogram, Scope: p.scope, } - measures, err := p.int64Resolver.HistogramAggregators(inst, boundaries) + measures, err := p.int64Resolver.HistogramAggregators(inst, allowedKeys, boundaries) return measures, errors.Join(aggError, err) } // lookup returns the resolved instrumentImpl. -func (p int64InstProvider) lookup(kind InstrumentKind, name, desc, u string) (*int64Inst, error) { +func (p int64InstProvider) lookup( + kind InstrumentKind, + name, desc, u string, + allowedKeys []attribute.Key, +) (*int64Inst, error) { return p.int64Insts.Lookup(instID{ Name: name, Description: desc, Unit: u, Kind: kind, }, func() (*int64Inst, error) { - aggs, err := p.aggs(kind, name, desc, u) + aggs, err := p.aggs(kind, name, desc, u, allowedKeys) return &int64Inst{measures: aggs}, err }) } // lookupHistogram returns the resolved instrumentImpl. -func (p int64InstProvider) lookupHistogram(name string, cfg metric.Int64HistogramConfig) (*int64Inst, error) { +func (p int64InstProvider) lookupHistogram( + name string, + cfg metric.Int64HistogramConfig, + allowedKeys []attribute.Key, +) (*int64Inst, error) { return p.int64Insts.Lookup(instID{ Name: name, Description: cfg.Description(), Unit: cfg.Unit(), Kind: InstrumentKindHistogram, }, func() (*int64Inst, error) { - aggs, err := p.histogramAggs(name, cfg) + aggs, err := p.histogramAggs(name, cfg, allowedKeys) return &int64Inst{measures: aggs}, err }) } @@ -694,7 +715,11 @@ func (p int64InstProvider) lookupHistogram(name string, cfg metric.Int64Histogra // float64InstProvider provides float64 OpenTelemetry instruments. type float64InstProvider struct{ *meter } -func (p float64InstProvider) aggs(kind InstrumentKind, name, desc, u string) ([]aggregate.Measure[float64], error) { +func (p float64InstProvider) aggs( + kind InstrumentKind, + name, desc, u string, + allowedKeys []attribute.Key, +) ([]aggregate.Measure[float64], error) { inst := Instrument{ Name: name, Description: desc, @@ -702,12 +727,13 @@ func (p float64InstProvider) aggs(kind InstrumentKind, name, desc, u string) ([] Kind: kind, Scope: p.scope, } - return p.float64Resolver.Aggregators(inst) + return p.float64Resolver.Aggregators(inst, allowedKeys) } func (p float64InstProvider) histogramAggs( name string, cfg metric.Float64HistogramConfig, + allowedKeys []attribute.Key, ) ([]aggregate.Measure[float64], error) { boundaries := cfg.ExplicitBucketBoundaries() aggError := AggregationExplicitBucketHistogram{Boundaries: boundaries}.err() @@ -722,32 +748,40 @@ func (p float64InstProvider) histogramAggs( Kind: InstrumentKindHistogram, Scope: p.scope, } - measures, err := p.float64Resolver.HistogramAggregators(inst, boundaries) + measures, err := p.float64Resolver.HistogramAggregators(inst, allowedKeys, boundaries) return measures, errors.Join(aggError, err) } // lookup returns the resolved instrumentImpl. -func (p float64InstProvider) lookup(kind InstrumentKind, name, desc, u string) (*float64Inst, error) { +func (p float64InstProvider) lookup( + kind InstrumentKind, + name, desc, u string, + allowedKeys []attribute.Key, +) (*float64Inst, error) { return p.float64Insts.Lookup(instID{ Name: name, Description: desc, Unit: u, Kind: kind, }, func() (*float64Inst, error) { - aggs, err := p.aggs(kind, name, desc, u) + aggs, err := p.aggs(kind, name, desc, u, allowedKeys) return &float64Inst{measures: aggs}, err }) } // lookupHistogram returns the resolved instrumentImpl. -func (p float64InstProvider) lookupHistogram(name string, cfg metric.Float64HistogramConfig) (*float64Inst, error) { +func (p float64InstProvider) lookupHistogram( + name string, + cfg metric.Float64HistogramConfig, + allowedKeys []attribute.Key, +) (*float64Inst, error) { return p.float64Insts.Lookup(instID{ Name: name, Description: cfg.Description(), Unit: cfg.Unit(), Kind: InstrumentKindHistogram, }, func() (*float64Inst, error) { - aggs, err := p.histogramAggs(name, cfg) + aggs, err := p.histogramAggs(name, cfg, allowedKeys) return &float64Inst{measures: aggs}, err }) } @@ -771,3 +805,18 @@ func (o float64Observer) Observe(val float64, opts ...metric.ObserveOption) { c := metric.NewObserveConfig(opts) o.observe(val, c.Attributes()) } + +func defaultAttributes[T any](opts []T) []attribute.Key { + var keys []attribute.Key + var found bool + for _, o := range opts { + if exp, ok := any(o).(interface{ AllowedKeys() []attribute.Key }); ok { + found = true + keys = append(keys, exp.AllowedKeys()...) + } + } + if found && keys == nil { + return []attribute.Key{} + } + return keys +} diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/periodic_reader.go b/vendor/go.opentelemetry.io/otel/sdk/metric/periodic_reader.go index d1efc9f37..c1c2e3e2f 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/periodic_reader.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/periodic_reader.go @@ -14,8 +14,9 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/sdk/metric/internal/observ" + "go.opentelemetry.io/otel/sdk/metric/internal/x" "go.opentelemetry.io/otel/sdk/metric/metricdata" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) // Default periodic reader timing. @@ -126,6 +127,9 @@ func NewPeriodicReader(exporter Exporter, options ...PeriodicReaderOption) *Peri }, }, } + if val, ok := x.MetricExportBatchSize.Lookup(); ok { + r.batcher = batcher{size: val} + } r.externalProducers.Store(conf.producers) go func() { @@ -164,6 +168,7 @@ type PeriodicReader struct { interval time.Duration timeout time.Duration + batcher batcher exporter Exporter flushCh chan chan error @@ -235,16 +240,28 @@ func (r *PeriodicReader) cardinalityLimit(kind InstrumentKind) (int, bool) { // collectAndExport gather all metric data related to the periodicReader r from // the SDK and exports it with r's exporter. func (r *PeriodicReader) collectAndExport(ctx context.Context) error { + originalCtx := ctx ctx, cancel := context.WithTimeoutCause(ctx, r.timeout, errors.New("reader collect and export timeout")) defer cancel() - // TODO (#3047): Use a sync.Pool or persistent pointer instead of allocating rm every Collect. rm := r.rmPool.Get().(*metricdata.ResourceMetrics) + defer func() { + *rm = metricdata.ResourceMetrics{} // erase fields to allow GC to collect them. + r.rmPool.Put(rm) + }() err := r.Collect(ctx, rm) if err == nil { - err = r.export(ctx, rm) + if r.batcher.size > 0 { + batches := r.batcher.splitResourceMetrics(rm) + for _, batch := range batches { + // The export timeout is applied individually to each batch by using + // the original context. + err = errors.Join(err, r.exportWithTimeout(originalCtx, batch)) + } + } else { + err = r.exporter.Export(ctx, rm) + } } - r.rmPool.Put(rm) return err } @@ -307,7 +324,10 @@ func (r *PeriodicReader) collect(ctx context.Context, p any, rm *metricdata.Reso } // export exports metric data m using r's exporter. -func (r *PeriodicReader) export(ctx context.Context, m *metricdata.ResourceMetrics) error { +func (r *PeriodicReader) exportWithTimeout(ctx context.Context, m *metricdata.ResourceMetrics) error { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeoutCause(ctx, r.timeout, errors.New("reader export timeout")) + defer cancel() return r.exporter.Export(ctx, m) } @@ -349,7 +369,9 @@ func (r *PeriodicReader) Shutdown(ctx context.Context) error { err := ErrReaderShutdown r.shutdownOnce.Do(func() { // Prioritize the ctx timeout if it is set. - if _, ok := ctx.Deadline(); !ok { + originalCtx := ctx + _, userProvidedContext := ctx.Deadline() + if !userProvidedContext { var cancel context.CancelFunc ctx, cancel = context.WithTimeoutCause(ctx, r.timeout, errors.New("reader shutdown timeout")) defer cancel() @@ -369,7 +391,24 @@ func (r *PeriodicReader) Shutdown(ctx context.Context) error { m := r.rmPool.Get().(*metricdata.ResourceMetrics) err = r.collect(ctx, ph, m) if err == nil { - err = r.export(ctx, m) + if r.batcher.size > 0 { + batches := r.batcher.splitResourceMetrics(m) + for _, batch := range batches { + if userProvidedContext { + // Do not apply the export timeout if the user passed a timeout to + // Shutdown(). + err = errors.Join(err, r.exporter.Export(ctx, batch)) + } else { + // The export timeout is applied individually to each batch by using + // the original context. + err = errors.Join(err, r.exportWithTimeout(originalCtx, batch)) + } + } + } else { + // Do not apply the export timeout if the user passed a timeout to + // Shutdown(). + err = r.exporter.Export(ctx, m) + } } r.rmPool.Put(m) } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/pipeline.go b/vendor/go.opentelemetry.io/otel/sdk/metric/pipeline.go index 34300a786..09d10eeb5 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/pipeline.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/pipeline.go @@ -11,6 +11,7 @@ import ( "sync" "sync/atomic" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/metric/embedded" "go.opentelemetry.io/otel/sdk/instrumentation" @@ -236,7 +237,11 @@ func newInserter[N int64 | float64](p *pipeline, vc *cache[string, instID]) *ins // // If an instrument is determined to use a Drop aggregation, that instrument is // not inserted nor returned. -func (i *inserter[N]) Instrument(inst Instrument, readerAggregation Aggregation) ([]aggregate.Measure[N], error) { +func (i *inserter[N]) Instrument( + inst Instrument, + allowedKeys []attribute.Key, + readerAggregation Aggregation, +) ([]aggregate.Measure[N], error) { var ( matched bool measures []aggregate.Measure[N] @@ -279,6 +284,12 @@ func (i *inserter[N]) Instrument(inst Instrument, readerAggregation Aggregation) Description: inst.Description, Unit: inst.Unit, } + // allowedKeys == nil indicates that the WithDefaultAttributes option was not passed, + // and all keys are allowed. An empty (non-nil) slice indicates that the option was passed + // with an empty set of keys, and no keys are allowed. + if allowedKeys != nil { + stream.AttributeFilter = attribute.NewAllowKeysFilter(allowedKeys...) + } in, _, e := i.cachedAggregator(inst.Scope, inst.Kind, stream, readerAggregation) if e != nil { if err == nil { @@ -388,6 +399,7 @@ func (i *inserter[N]) cachedAggregator( b := aggregate.Builder[N]{ Temporality: i.pipeline.reader.temporality(kind), ReservoirFunc: reservoirFunc[N]( + kind, stream.ExemplarReservoirProviderSelector(stream.Aggregation), i.pipeline.exemplarFilter, ), @@ -661,12 +673,12 @@ func newResolver[N int64 | float64](p pipelines, vc *cache[string, instID]) reso // Aggregators returns the Aggregators that must be updated by the instrument // defined by key. -func (r resolver[N]) Aggregators(id Instrument) ([]aggregate.Measure[N], error) { +func (r resolver[N]) Aggregators(id Instrument, allowedKeys []attribute.Key) ([]aggregate.Measure[N], error) { var measures []aggregate.Measure[N] var err error for _, i := range r.inserters { - in, e := i.Instrument(id, i.readerDefaultAggregation(id.Kind)) + in, e := i.Instrument(id, allowedKeys, i.readerDefaultAggregation(id.Kind)) if e != nil { err = errors.Join(err, e) } @@ -678,7 +690,11 @@ func (r resolver[N]) Aggregators(id Instrument) ([]aggregate.Measure[N], error) // HistogramAggregators returns the histogram Aggregators that must be updated by the instrument // defined by key. If boundaries were provided on instrument instantiation, those take precedence // over boundaries provided by the reader. -func (r resolver[N]) HistogramAggregators(id Instrument, boundaries []float64) ([]aggregate.Measure[N], error) { +func (r resolver[N]) HistogramAggregators( + id Instrument, + allowedKeys []attribute.Key, + boundaries []float64, +) ([]aggregate.Measure[N], error) { var measures []aggregate.Measure[N] var err error @@ -688,7 +704,7 @@ func (r resolver[N]) HistogramAggregators(id Instrument, boundaries []float64) ( histAgg.Boundaries = boundaries agg = histAgg } - in, e := i.Instrument(id, agg) + in, e := i.Instrument(id, allowedKeys, agg) if e != nil { err = errors.Join(err, e) } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/provider.go b/vendor/go.opentelemetry.io/otel/sdk/metric/provider.go index b0a6ec580..fd4cdccd0 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/provider.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/provider.go @@ -47,7 +47,8 @@ func NewMeterProvider(options ...Option) *MeterProvider { shutdown: sdown, } // Log after creation so all readers show correctly they are registered. - global.Info("MeterProvider created", + global.Info( + "MeterProvider created", "Resource", conf.res, "Readers", conf.readers, "Views", len(conf.views), @@ -82,7 +83,8 @@ func (mp *MeterProvider) Meter(name string, options ...metric.MeterOption) metri Attributes: c.InstrumentationAttributes(), } - global.Info("Meter created", + global.Info( + "Meter created", "Name", s.Name, "Version", s.Version, "SchemaURL", s.SchemaURL, diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/reader.go b/vendor/go.opentelemetry.io/otel/sdk/metric/reader.go index 99079dd27..556afcea6 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/reader.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/reader.go @@ -183,6 +183,15 @@ type AggregationSelector func(InstrumentKind) Aggregation // mapping: Counter ⇨ Sum, Observable Counter ⇨ Sum, UpDownCounter ⇨ Sum, // Observable UpDownCounter ⇨ Sum, Observable Gauge ⇨ LastValue, // Histogram ⇨ ExplicitBucketHistogram. +// +// The default ExplicitBucketHistogram boundaries are designed for +// millisecond-scale latency values. Boundaries are interpreted relative to the +// values recorded for the instrument and are not rescaled when an instrument is +// created with a different unit (e.g. via +// [go.opentelemetry.io/otel/metric.WithUnit]). Instrumentation authors should +// supply appropriate boundaries per instrument via +// [go.opentelemetry.io/otel/metric.WithExplicitBucketBoundaries]; end users +// can also override boundaries for a specific instrument with a [View]. func DefaultAggregationSelector(ik InstrumentKind) Aggregation { switch ik { case InstrumentKindCounter, diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/splitmetrics.go b/vendor/go.opentelemetry.io/otel/sdk/metric/splitmetrics.go new file mode 100644 index 000000000..a7931412b --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/splitmetrics.go @@ -0,0 +1,191 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package metric // import "go.opentelemetry.io/otel/sdk/metric" + +import ( + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// batcher splits metrics into batches. +type batcher struct { + size int +} + +// splitResourceMetrics splits a metricdata.ResourceMetrics into multiple ResourceMetrics, sequentially, +// ensuring no ResourceMetrics has more than `size` data points. It does not mutate the `src` object. +func (b batcher) splitResourceMetrics(src *metricdata.ResourceMetrics) []*metricdata.ResourceMetrics { + if b.size <= 0 || len(src.ScopeMetrics) == 0 { + return []*metricdata.ResourceMetrics{src} + } + var batches []*metricdata.ResourceMetrics + var currentBatch *metricdata.ResourceMetrics + currentPoints := 0 + + for i := 0; i < len(src.ScopeMetrics); i++ { + sm := src.ScopeMetrics[i] + + take := b.size - currentPoints + smChunks := b.splitScopeMetrics(sm, take) + + for _, chunk := range smChunks { + if currentBatch == nil { + currentBatch = &metricdata.ResourceMetrics{Resource: src.Resource} + batches = append(batches, currentBatch) + } + currentBatch.ScopeMetrics = append(currentBatch.ScopeMetrics, chunk) + currentPoints += scopeMetricsDPC(chunk) + + if currentPoints == b.size { + currentBatch = nil + currentPoints = 0 + } + } + } + return batches +} + +// splitScopeMetrics splits a metricdata.ScopeMetrics into chunks. The first chunk will have at most firstSize data points. +func (b batcher) splitScopeMetrics(sm metricdata.ScopeMetrics, firstSize int) []metricdata.ScopeMetrics { + smPoints := scopeMetricsDPC(sm) + if smPoints <= firstSize { + return []metricdata.ScopeMetrics{sm} + } + + var chunks []metricdata.ScopeMetrics + var currentChunk *metricdata.ScopeMetrics + currentPoints := 0 + targetSize := firstSize + + for i := 0; i < len(sm.Metrics); i++ { + m := sm.Metrics[i] + + take := targetSize - currentPoints + mChunks := b.splitMetric(m, take) + + for _, mc := range mChunks { + if currentChunk == nil { + chunks = append(chunks, metricdata.ScopeMetrics{Scope: sm.Scope}) + currentChunk = &chunks[len(chunks)-1] + } + currentChunk.Metrics = append(currentChunk.Metrics, mc) + currentPoints += metricDPC(mc) + + if currentPoints == targetSize { + currentChunk = nil + currentPoints = 0 + targetSize = b.size + } + } + } + return chunks +} + +// splitMetric splits a metricdata.Metrics into chunks. The first chunk will have at most firstSize data points. +func (b batcher) splitMetric(m metricdata.Metrics, firstSize int) []metricdata.Metrics { + mPoints := metricDPC(m) + if mPoints <= firstSize { + return []metricdata.Metrics{m} + } + + var chunks []metricdata.Metrics + mRemaining := mPoints + mOffset := 0 + take := firstSize + + for mRemaining > 0 { + if take > mRemaining { + take = mRemaining + } + chunks = append(chunks, copyMetricData(m, mOffset, take)) + mRemaining -= take + mOffset += take + take = b.size + } + return chunks +} + +// copyMetricData creates a copy of the metricdata.Metrics with the specified offset and number of datapoints to take. +func copyMetricData(m metricdata.Metrics, offset, take int) metricdata.Metrics { + dest := metricdata.Metrics{ + Name: m.Name, + Description: m.Description, + Unit: m.Unit, + } + switch a := m.Data.(type) { + case metricdata.Gauge[int64]: + dest.Data = metricdata.Gauge[int64]{DataPoints: a.DataPoints[offset : offset+take]} + case metricdata.Gauge[float64]: + dest.Data = metricdata.Gauge[float64]{DataPoints: a.DataPoints[offset : offset+take]} + case metricdata.Sum[int64]: + dest.Data = metricdata.Sum[int64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + IsMonotonic: a.IsMonotonic, + } + case metricdata.Sum[float64]: + dest.Data = metricdata.Sum[float64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + IsMonotonic: a.IsMonotonic, + } + case metricdata.Histogram[int64]: + dest.Data = metricdata.Histogram[int64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + } + case metricdata.Histogram[float64]: + dest.Data = metricdata.Histogram[float64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + } + case metricdata.ExponentialHistogram[int64]: + dest.Data = metricdata.ExponentialHistogram[int64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + } + case metricdata.ExponentialHistogram[float64]: + dest.Data = metricdata.ExponentialHistogram[float64]{ + DataPoints: a.DataPoints[offset : offset+take], + Temporality: a.Temporality, + } + case metricdata.Summary: + dest.Data = metricdata.Summary{DataPoints: a.DataPoints[offset : offset+take]} + } + return dest +} + +// scopeMetricsDPC calculates the total number of data points in the metricdata.ScopeMetrics. +func scopeMetricsDPC(sm metricdata.ScopeMetrics) int { + dataPointCount := 0 + ms := sm.Metrics + for k := range ms { + dataPointCount += metricDPC(ms[k]) + } + return dataPointCount +} + +// metricDPC calculates the total number of data points in the metricdata.Metrics. +func metricDPC(m metricdata.Metrics) int { + switch a := m.Data.(type) { + case metricdata.Gauge[int64]: + return len(a.DataPoints) + case metricdata.Gauge[float64]: + return len(a.DataPoints) + case metricdata.Sum[int64]: + return len(a.DataPoints) + case metricdata.Sum[float64]: + return len(a.DataPoints) + case metricdata.Histogram[int64]: + return len(a.DataPoints) + case metricdata.Histogram[float64]: + return len(a.DataPoints) + case metricdata.ExponentialHistogram[int64]: + return len(a.DataPoints) + case metricdata.ExponentialHistogram[float64]: + return len(a.DataPoints) + case metricdata.Summary: + return len(a.DataPoints) + } + return 0 +} diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/version.go b/vendor/go.opentelemetry.io/otel/sdk/metric/version.go index 26752be7d..3e1bea4b4 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/version.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/version.go @@ -5,5 +5,5 @@ package metric // import "go.opentelemetry.io/otel/sdk/metric" // version is the current release version of the metric SDK in use. func version() string { - return "1.43.0" + return "1.44.0" } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/view.go b/vendor/go.opentelemetry.io/otel/sdk/metric/view.go index 630890f42..13fb7143c 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/metric/view.go +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/view.go @@ -22,6 +22,10 @@ var ( // should be collected for certain instruments. It returns true and the exact // Stream to use for matching Instruments. Otherwise, if the view does not // match, false is returned. +// +// Note that attributes filtered out by a View may still appear on Exemplars, +// because Exemplars are recorded with the dropped measurement attributes +// when View attribute filtering is applied. type View func(Instrument) (Stream, bool) // NewView returns a View that applies the Stream mask for all instruments that diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/builtin.go b/vendor/go.opentelemetry.io/otel/sdk/resource/builtin.go index 04f15fcd2..28823edd5 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/builtin.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/builtin.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) type ( @@ -79,7 +79,7 @@ func (sd stringDetector) Detect(context.Context) (*Resource, error) { } a := sd.K.String(value) if !a.Valid() { - return nil, fmt.Errorf("invalid attribute: %q -> %q", a.Key, a.Value.Emit()) + return nil, fmt.Errorf("invalid attribute: %q -> %q", a.Key, a.Value.String()) } return NewWithAttributes(sd.schemaURL, sd.K.String(value)), nil } diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/container.go b/vendor/go.opentelemetry.io/otel/sdk/resource/container.go index e977ff1c4..ce03e24c4 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/container.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/container.go @@ -11,7 +11,7 @@ import ( "os" "regexp" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) type containerIDProvider func() (string, error) diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/env.go b/vendor/go.opentelemetry.io/otel/sdk/resource/env.go index bc0e5c19e..ac5691c08 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/env.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/env.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) const ( diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/host_id.go b/vendor/go.opentelemetry.io/otel/sdk/resource/host_id.go index 755c08242..cb38fa1a8 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/host_id.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/host_id.go @@ -8,7 +8,7 @@ import ( "errors" "strings" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) type hostIDProvider func() (string, error) diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/host_id_exec.go b/vendor/go.opentelemetry.io/otel/sdk/resource/host_id_exec.go index d9e5d1a8f..e239ead02 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/host_id_exec.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/host_id_exec.go @@ -5,10 +5,13 @@ package resource // import "go.opentelemetry.io/otel/sdk/resource" -import "os/exec" +import ( + "context" + "os/exec" +) func execCommand(name string, arg ...string) (string, error) { - cmd := exec.Command(name, arg...) + cmd := exec.CommandContext(context.Background(), name, arg...) b, err := cmd.Output() if err != nil { return "", err diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/os.go b/vendor/go.opentelemetry.io/otel/sdk/resource/os.go index f5682cad4..4c0def414 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/os.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/os.go @@ -8,7 +8,7 @@ import ( "strings" "go.opentelemetry.io/otel/attribute" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) type osDescriptionProvider func() (string, error) diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/os_unix.go b/vendor/go.opentelemetry.io/otel/sdk/resource/os_unix.go index 6c50ab686..1cd87c398 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/os_unix.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/os_unix.go @@ -55,7 +55,8 @@ func uname() (string, error) { return "", err } - return fmt.Sprintf("%s %s %s %s %s", + return fmt.Sprintf( + "%s %s %s %s %s", unix.ByteSliceToString(utsName.Sysname[:]), unix.ByteSliceToString(utsName.Nodename[:]), unix.ByteSliceToString(utsName.Release[:]), diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/os_windows.go b/vendor/go.opentelemetry.io/otel/sdk/resource/os_windows.go index a6a5a53c0..ebd50826d 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/os_windows.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/os_windows.go @@ -16,7 +16,8 @@ import ( // resembles the one displayed by the Version Reporter Applet (winver.exe). func platformOSDescription() (string, error) { k, err := registry.OpenKey( - registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) + registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE, + ) if err != nil { return "", err } @@ -37,7 +38,8 @@ func platformOSDescription() (string, error) { displayVersion += " " } - return fmt.Sprintf("%s %s(%s) [Version %s.%s.%s.%s]", + return fmt.Sprintf( + "%s %s(%s) [Version %s.%s.%s.%s]", productName, displayVersion, releaseID, diff --git a/vendor/go.opentelemetry.io/otel/sdk/resource/process.go b/vendor/go.opentelemetry.io/otel/sdk/resource/process.go index 99dce64f6..b015f8233 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/resource/process.go +++ b/vendor/go.opentelemetry.io/otel/sdk/resource/process.go @@ -11,7 +11,7 @@ import ( "path/filepath" "runtime" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) type ( @@ -164,7 +164,8 @@ func (processRuntimeVersionDetector) Detect(context.Context) (*Resource, error) // Detect returns a *Resource that describes the runtime of this process. func (processRuntimeDescriptionDetector) Detect(context.Context) (*Resource, error) { runtimeDescription := fmt.Sprintf( - "go version %s %s/%s", runtimeVersion(), runtimeOS(), runtimeArch()) + "go version %s %s/%s", runtimeVersion(), runtimeOS(), runtimeArch(), + ) return NewWithAttributes( semconv.SchemaURL, diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/batch_span_processor.go b/vendor/go.opentelemetry.io/otel/sdk/trace/batch_span_processor.go index 32854b14a..f9f2a6c2c 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/batch_span_processor.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/batch_span_processor.go @@ -163,19 +163,21 @@ func (bsp *batchSpanProcessor) Shutdown(ctx context.Context) error { bsp.stopOnce.Do(func() { bsp.stopped.Store(true) wait := make(chan struct{}) + // exportErr is written by the goroutine before closing wait. + // It is only read in the <-wait case, so there is no race. + var exportErr error go func() { close(bsp.stopCh) bsp.stopWait.Wait() if bsp.e != nil { - if err := bsp.e.Shutdown(ctx); err != nil { - otel.Handle(err) - } + exportErr = bsp.e.Shutdown(ctx) } close(wait) }() - // Wait until the wait group is done or the context is cancelled + // Wait until the channel is ready or the context is canceled. select { case <-wait: + err = exportErr case <-ctx.Done(): err = ctx.Err() } diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/batch_span_processor.go b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/batch_span_processor.go index c31e03aa0..c725ebf37 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/batch_span_processor.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/batch_span_processor.go @@ -13,8 +13,8 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/internal/x" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) const ( diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/simple_span_processor.go b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/simple_span_processor.go index 0e77cd953..cf4b5d481 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/simple_span_processor.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/simple_span_processor.go @@ -13,8 +13,8 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/internal/x" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" ) var measureAttrsPool = sync.Pool{ @@ -86,6 +86,7 @@ func (ssp *SSP) addOption(err error) []metric.AddOption { } attrs := measureAttrsPool.Get().(*[]attribute.KeyValue) defer func() { + clear(*attrs) *attrs = (*attrs)[:0] // reset the slice for reuse measureAttrsPool.Put(attrs) }() diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/tracer.go b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/tracer.go index 560d316f2..5aae89e88 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/tracer.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/internal/observ/tracer.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/internal/x" - "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" + "go.opentelemetry.io/otel/semconv/v1.41.0/otelconv" "go.opentelemetry.io/otel/trace" ) diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/provider.go b/vendor/go.opentelemetry.io/otel/sdk/trace/provider.go index cd40d299d..9d90c2eda 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/provider.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/provider.go @@ -82,6 +82,10 @@ type TracerProvider struct { var _ trace.TracerProvider = &TracerProvider{} +type experimentalOption interface { + Experimental() +} + // NewTracerProvider returns a new and configured TracerProvider. // // By default the returned TracerProvider is configured with: @@ -99,6 +103,9 @@ func NewTracerProvider(opts ...TracerProviderOption) *TracerProvider { o = applyTracerProviderEnvConfigs(o) for _, opt := range opts { + if _, ok := opt.(experimentalOption); ok { + continue + } o = opt.apply(o) } @@ -310,7 +317,7 @@ func (p *TracerProvider) Shutdown(ctx context.Context) error { } func (p *TracerProvider) getSpanProcessors() spanProcessorStates { - return *(p.spanProcessors.Load()) + return *p.spanProcessors.Load() } // TracerProviderOption configures a TracerProvider. diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/sampling.go b/vendor/go.opentelemetry.io/otel/sdk/trace/sampling.go index 845e292c2..5a4c74318 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/sampling.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/sampling.go @@ -297,8 +297,9 @@ func (pb parentBased) ShouldSample(p SamplingParameters) SamplingResult { } func (pb parentBased) Description() string { - return fmt.Sprintf("ParentBased{root:%s,remoteParentSampled:%s,"+ - "remoteParentNotSampled:%s,localParentSampled:%s,localParentNotSampled:%s}", + return fmt.Sprintf( + "ParentBased{root:%s,remoteParentSampled:%s,"+ + "remoteParentNotSampled:%s,localParentSampled:%s,localParentNotSampled:%s}", pb.root.Description(), pb.config.remoteParentSampled.Description(), pb.config.remoteParentNotSampled.Description(), diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/span.go b/vendor/go.opentelemetry.io/otel/sdk/trace/span.go index 7d55ce1dc..d1d9af29d 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/span.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/span.go @@ -20,7 +20,7 @@ import ( "go.opentelemetry.io/otel/internal/global" "go.opentelemetry.io/otel/sdk/instrumentation" "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/embedded" ) @@ -346,10 +346,12 @@ func (s *recordingSpan) addOverCapAttrs(limit int, attrs []attribute.KeyValue) { } } -// truncateAttr returns a truncated version of attr. Only string and string -// slice attribute values are truncated. String values are truncated to at -// most a length of limit. Each string slice value is truncated in this fashion -// (the slice length itself is unaffected). +// truncateAttr returns a truncated version of attr. Only string, string +// slice, byte slice, and slice attribute values are truncated. String values are truncated +// to at most a length of limit. Each string slice value is truncated in this +// fashion (the slice length itself is unaffected), and byte slice values are truncated to at most +// limit bytes. For slice attribute values, the limit is applied to each +// element recursively. // // No truncation is performed for a negative limit. func truncateAttr(limit int, attr attribute.KeyValue) attribute.KeyValue { @@ -366,10 +368,95 @@ func truncateAttr(limit int, attr attribute.KeyValue) attribute.KeyValue { v[i] = truncate(limit, v[i]) } return attr.Key.StringSlice(v) + case attribute.BYTESLICE: + v := attr.Value.AsString() + if len(v) > limit { + return attr.Key.ByteSlice([]byte(v[:limit])) + } + return attr + case attribute.SLICE: + v := attr.Value.AsSlice() + if !slices.ContainsFunc(v, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return attr + } + newV := make([]attribute.Value, len(v)) + for i, elem := range v { + newV[i] = truncateValue(limit, elem) + } + return attr.Key.Slice(newV...) } return attr } +// truncateValue returns a truncated version of v. Only string, string slice, +// byte slice, and (recursively) slice values are modified. +// +// No truncation is performed for a negative limit. +func truncateValue(limit int, v attribute.Value) attribute.Value { + switch v.Type() { + case attribute.STRING: + return attribute.StringValue(truncate(limit, v.AsString())) + case attribute.STRINGSLICE: + ss := v.AsStringSlice() + for i := range ss { + ss[i] = truncate(limit, ss[i]) + } + return attribute.StringSliceValue(ss) + + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids allocating the full slice before truncation. + s := v.AsString() + if limit >= 0 && len(s) > limit { + return attribute.ByteSliceValue([]byte(s[:limit])) + } + case attribute.SLICE: + sl := v.AsSlice() + if !slices.ContainsFunc(sl, func(e attribute.Value) bool { return needsTruncation(limit, e) }) { + return v + } + newSl := make([]attribute.Value, len(sl)) + for i, elem := range sl { + newSl[i] = truncateValue(limit, elem) + } + return attribute.SliceValue(newSl...) + } + return v +} + +// stringNeedsTruncation reports whether s would be modified by truncate for the +// given limit. +func stringNeedsTruncation(limit int, s string) bool { + if limit < 0 || len(s) <= limit { + return false + } + return utf8.RuneCountInString(s) > limit || !utf8.ValidString(s) +} + +// needsTruncation reports whether v would be modified by truncateValue for the +// given limit. +func needsTruncation(limit int, v attribute.Value) bool { + switch v.Type() { + case attribute.STRING: + return stringNeedsTruncation(limit, v.AsString()) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids memory allocation. + if limit >= 0 && len(v.AsString()) > limit { + return true + } + case attribute.STRINGSLICE: + for _, s := range v.AsStringSlice() { + if stringNeedsTruncation(limit, s) { + return true + } + } + case attribute.SLICE: + return slices.ContainsFunc(v.AsSlice(), func(e attribute.Value) bool { return needsTruncation(limit, e) }) + } + return false +} + // truncate returns a truncated version of s such that it contains less than // the limit number of characters. Truncation is applied by returning the limit // number of valid characters contained in s. diff --git a/vendor/go.opentelemetry.io/otel/sdk/trace/span_limits.go b/vendor/go.opentelemetry.io/otel/sdk/trace/span_limits.go index 321d97430..348ee0e80 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/trace/span_limits.go +++ b/vendor/go.opentelemetry.io/otel/sdk/trace/span_limits.go @@ -35,8 +35,10 @@ const ( type SpanLimits struct { // AttributeValueLengthLimit is the maximum allowed attribute value length. // - // This limit only applies to string and string slice attribute values. - // Any string longer than this value will be truncated to this length. + // This limit only applies to string, string slice, byte slice, and slice attribute + // values. Any string and byte slice longer than this value will be truncated to this + // length. For slice attribute values, the limit is applied to each string and byte slice + // element recursively. // // Setting this to a negative value means no limit is applied. AttributeValueLengthLimit int diff --git a/vendor/go.opentelemetry.io/otel/sdk/version.go b/vendor/go.opentelemetry.io/otel/sdk/version.go index 766731dd2..218dce1f5 100644 --- a/vendor/go.opentelemetry.io/otel/sdk/version.go +++ b/vendor/go.opentelemetry.io/otel/sdk/version.go @@ -6,5 +6,5 @@ package sdk // import "go.opentelemetry.io/otel/sdk" // Version is the current release version of the OpenTelemetry SDK in use. func Version() string { - return "1.43.0" + return "1.44.0" } diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go b/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go index b6b27498f..2fcab2435 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go @@ -1447,9 +1447,11 @@ func AWSExtendedRequestID(val string) attribute.KeyValue { // AWSKinesisStreamName returns an attribute KeyValue conforming to the // "aws.kinesis.stream_name" semantic conventions. It represents the name of the // AWS Kinesis [stream] the request refers to. Corresponds to the `--stream-name` -// parameter of the Kinesis [describe-stream] operation. +// +// parameter of the Kinesis [describe-stream] operation. // // [stream]: https://docs.aws.amazon.com/streams/latest/dev/introduction.html +// // [describe-stream]: https://docs.aws.amazon.com/cli/latest/reference/kinesis/describe-stream.html func AWSKinesisStreamName(val string) attribute.KeyValue { return AWSKinesisStreamNameKey.String(val) @@ -1459,7 +1461,8 @@ func AWSKinesisStreamName(val string) attribute.KeyValue { // "aws.lambda.invoked_arn" semantic conventions. It represents the full invoked // ARN as provided on the `Context` passed to the function ( // `Lambda-Runtime-Invoked-Function-Arn` header on the `/runtime/invocation/next` -// applicable). +// +// applicable). func AWSLambdaInvokedARN(val string) attribute.KeyValue { return AWSLambdaInvokedARNKey.String(val) } @@ -2635,7 +2638,8 @@ func CloudRegion(val string) attribute.KeyValue { // "cloud.resource_id" semantic conventions. It represents the cloud // provider-specific native identifier of the monitored cloud resource (e.g. an // [ARN] on AWS, a [fully qualified resource ID] on Azure, a [full resource name] -// on GCP). +// +// on GCP). // // [ARN]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html // [fully qualified resource ID]: https://learn.microsoft.com/rest/api/resources/resources/get-by-id @@ -15190,4 +15194,4 @@ func ZOSSmfID(val string) attribute.KeyValue { // to which the z/OS system belongs too. func ZOSSysplexName(val string) attribute.KeyValue { return ZOSSysplexNameKey.String(val) -} \ No newline at end of file +} diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/README.md b/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/README.md deleted file mode 100644 index c51b7fb7b..000000000 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Semconv v1.40.0 - -[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/semconv/v1.40.0)](https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.40.0) diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/MIGRATION.md b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/MIGRATION.md similarity index 63% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/MIGRATION.md rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/MIGRATION.md index e246b1692..ba52cadf7 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/MIGRATION.md +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/MIGRATION.md @@ -1,7 +1,7 @@ -# Migration from v1.39.0 to v1.40.0 +# Migration from v1.40.0 to v1.41.0 -The `go.opentelemetry.io/otel/semconv/v1.40.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.39.0` with the following exceptions. +The `go.opentelemetry.io/otel/semconv/v1.41.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.40.0` with the following exceptions. ## Removed @@ -11,17 +11,7 @@ Refer to the [OpenTelemetry Semantic Conventions documentation] for deprecation If the type is not listed in the documentation as deprecated, it has been removed in this version due to lack of applicability or use. If you use any of these non-deprecated declarations in your Go application, please [open an issue] describing your use-case. -- `ErrorMessage` -- `ErrorMessageKey` -- `RPCMessageCompressedSize` -- `RPCMessageCompressedSizeKey` -- `RPCMessageID` -- `RPCMessageIDKey` -- `RPCMessageTypeKey` -- `RPCMessageTypeReceived` -- `RPCMessageTypeSent` -- `RPCMessageUncompressedSize` -- `RPCMessageUncompressedSizeKey` +- `DeploymentEnvironmentName` [OpenTelemetry Semantic Conventions documentation]: https://github.com/open-telemetry/semantic-conventions [open an issue]: https://github.com/open-telemetry/opentelemetry-go/issues/new?template=Blank+issue diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/README.md b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/README.md new file mode 100644 index 000000000..8353bb715 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/README.md @@ -0,0 +1,3 @@ +# Semconv v1.41.0 + +[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/semconv/v1.41.0)](https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.41.0) diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/attribute_group.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/attribute_group.go similarity index 96% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/attribute_group.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/attribute_group.go index ee6b1f79d..7cee08680 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/attribute_group.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/attribute_group.go @@ -3,7 +3,7 @@ // Code generated from semantic convention specification. DO NOT EDIT. -package semconv // import "go.opentelemetry.io/otel/semconv/v1.40.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.41.0" import "go.opentelemetry.io/otel/attribute" @@ -950,7 +950,7 @@ const ( // of the [AWS Lambda EvenSource Mapping]. An event source is mapped to a lambda // function. It's contents are read by Lambda and used to trigger a function. // This isn't available in the lambda execution context or the lambda runtime - // environtment. This is going to be populated by the AWS SDK for each language + // environment. This is going to be populated by the AWS SDK for each language // when that UUID is present. Some of these operations are // Create/Delete/Get/List/Update EventSourceMapping. // @@ -1186,7 +1186,7 @@ const ( // AWSSecretsmanagerSecretARNKey is the attribute Key conforming to the // "aws.secretsmanager.secret.arn" semantic conventions. It represents the ARN - // of the Secret stored in the Secrets Mangger. + // of the Secret stored in the Secrets Manager. // // Type: string // RequirementLevel: Recommended @@ -1515,7 +1515,7 @@ func AWSLambdaInvokedARN(val string) attribute.KeyValue { // of the [AWS Lambda EvenSource Mapping]. An event source is mapped to a lambda // function. It's contents are read by Lambda and used to trigger a function. // This isn't available in the lambda execution context or the lambda runtime -// environtment. This is going to be populated by the AWS SDK for each language +// environment. This is going to be populated by the AWS SDK for each language // when that UUID is present. Some of these operations are // Create/Delete/Get/List/Update EventSourceMapping. // @@ -1609,7 +1609,7 @@ func AWSS3UploadID(val string) attribute.KeyValue { // AWSSecretsmanagerSecretARN returns an attribute KeyValue conforming to the // "aws.secretsmanager.secret.arn" semantic conventions. It represents the ARN of -// the Secret stored in the Secrets Mangger. +// the Secret stored in the Secrets Manager. func AWSSecretsmanagerSecretARN(val string) attribute.KeyValue { return AWSSecretsmanagerSecretARNKey.String(val) } @@ -2196,6 +2196,11 @@ const ( // Stability: Development // // Examples: "12097" + // Note: For a given pipeline run and task, the `cicd.pipeline.task.run.id` MUST + // be unique within that run. For the same task across different runs of the + // same pipeline, the `cicd.pipeline.task.run.id` MAY remain the same, enabling + // correlation of `cicd.pipeline.task.run.result` values across multiple + // pipeline runs. CICDPipelineTaskRunIDKey = attribute.Key("cicd.pipeline.task.run.id") // CICDPipelineTaskRunResultKey is the attribute Key conforming to the @@ -3431,7 +3436,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "a3bf90e006b2" // @@ -3467,7 +3472,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "gcr.io/opentelemetry/operator" ContainerImageNameKey = attribute.Key("container.image.name") @@ -3478,7 +3483,7 @@ const ( // // Type: string[] // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: // "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", @@ -3497,7 +3502,7 @@ const ( // // Type: string[] // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "v1.27.1", "3.5.7-0" // @@ -3518,7 +3523,7 @@ const ( // ContainerRuntimeDescriptionKey is the attribute Key conforming to the // "container.runtime.description" semantic conventions. It represents a // description about the runtime which could include, for example details about - // the CRI/API version being used or other customisations. + // the CRI/API version being used or other customizations. // // Type: string // RequirementLevel: Recommended @@ -3649,7 +3654,7 @@ func ContainerName(val string) attribute.KeyValue { // ContainerRuntimeDescription returns an attribute KeyValue conforming to the // "container.runtime.description" semantic conventions. It represents a // description about the runtime which could include, for example details about -// the CRI/API version being used or other customisations. +// the CRI/API version being used or other customizations. func ContainerRuntimeDescription(val string) attribute.KeyValue { return ContainerRuntimeDescriptionKey.String(val) } @@ -4260,9 +4265,9 @@ const ( // "deployment.environment.name" semantic conventions. It represents the name of // the [deployment environment] (aka deployment tier). // - // Type: string + // Type: Enum // RequirementLevel: Recommended - // Stability: Development + // Stability: Stable // // Examples: "staging", "production" // Note: `deployment.environment.name` does not affect the uniqueness @@ -4312,15 +4317,6 @@ const ( DeploymentStatusKey = attribute.Key("deployment.status") ) -// DeploymentEnvironmentName returns an attribute KeyValue conforming to the -// "deployment.environment.name" semantic conventions. It represents the name of -// the [deployment environment] (aka deployment tier). -// -// [deployment environment]: https://wikipedia.org/wiki/Deployment_environment -func DeploymentEnvironmentName(val string) attribute.KeyValue { - return DeploymentEnvironmentNameKey.String(val) -} - // DeploymentID returns an attribute KeyValue conforming to the "deployment.id" // semantic conventions. It represents the id of the deployment. func DeploymentID(val string) attribute.KeyValue { @@ -4334,6 +4330,22 @@ func DeploymentName(val string) attribute.KeyValue { return DeploymentNameKey.String(val) } +// Enum values for deployment.environment.name +var ( + // Production environment + // Stability: stable + DeploymentEnvironmentNameProduction = DeploymentEnvironmentNameKey.String("production") + // Staging environment + // Stability: stable + DeploymentEnvironmentNameStaging = DeploymentEnvironmentNameKey.String("staging") + // Testing environment + // Stability: stable + DeploymentEnvironmentNameTest = DeploymentEnvironmentNameKey.String("test") + // Development environment + // Stability: stable + DeploymentEnvironmentNameDevelopment = DeploymentEnvironmentNameKey.String("development") +) + // Enum values for deployment.status var ( // failed @@ -4645,6 +4657,12 @@ const ( // When `error.type` is set to a type (e.g., an exception type), its // canonical class name identifying the type within the artifact SHOULD be used. // + // If the recorded error type is a wrapper that is not meaningful for + // failure classification, instrumentation MAY use the type of the inner + // error instead. For example, in Go, errors created with `fmt.Errorf` + // using `%w` MAY be unwrapped when the wrapper type does not help + // classify the failure. + // // Instrumentations SHOULD document the list of errors they report. // // The cardinality of `error.type` within one instrumentation library SHOULD be @@ -4718,6 +4736,11 @@ const ( // Stability: Stable // // Examples: "java.net.ConnectException", "OSError" + // Note: If the recorded exception type is a wrapper that is not meaningful for + // failure classification, instrumentation MAY use the type of the inner + // exception instead. For example, in Go, errors created with `fmt.Errorf` + // using `%w` MAY be unwrapped when the wrapper type does not help + // classify the failure. ExceptionTypeKey = attribute.Key("exception.type") ) @@ -6667,6 +6690,17 @@ const ( // Examples: "forest", "lived" GenAIRequestStopSequencesKey = attribute.Key("gen_ai.request.stop_sequences") + // GenAIRequestStreamKey is the attribute Key conforming to the + // "gen_ai.request.stream" semantic conventions. It represents the indicates + // whether the GenAI request was made in streaming mode. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + GenAIRequestStreamKey = attribute.Key("gen_ai.request.stream") + // GenAIRequestTemperatureKey is the attribute Key conforming to the // "gen_ai.request.temperature" semantic conventions. It represents the // temperature setting for the GenAI request. @@ -6734,6 +6768,19 @@ const ( // Examples: "gpt-4-0613" GenAIResponseModelKey = attribute.Key("gen_ai.response.model") + // GenAIResponseTimeToFirstChunkKey is the attribute Key conforming to the + // "gen_ai.response.time_to_first_chunk" semantic conventions. It represents the + // time to first chunk in a streaming response, measured from request issuance, + // in seconds. The value is measured from when the client issues the generation + // request to when the first chunk is received in the response stream. + // + // Type: double + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: 0.5, 1.2 + GenAIResponseTimeToFirstChunkKey = attribute.Key("gen_ai.response.time_to_first_chunk") + // GenAIRetrievalDocumentsKey is the attribute Key conforming to the // "gen_ai.retrieval.documents" semantic conventions. It represents the // documents retrieved. @@ -6875,7 +6922,7 @@ const ( // GenAIToolDefinitionsKey is the attribute Key conforming to the // "gen_ai.tool.definitions" semantic conventions. It represents the list of - // source system tool definitions available to the GenAI agent or model. + // tool definitions available to the GenAI agent or model. // // Type: any // RequirementLevel: Recommended @@ -6887,19 +6934,18 @@ const ( // "description": "The city and state, e.g. San Francisco, CA"\n },\n "unit": // {\n "type": "string",\n "enum": [\n "celsius",\n "fahrenheit"\n ]\n }\n },\n // "required": [\n "location",\n "unit"\n ]\n }\n }\n]\n" - // Note: The value of this attribute matches source system tool definition - // format. + // Note: Instrumentations MUST follow [Tool Definitions JSON Schema]. // - // It's expected to be an array of objects where each object represents a tool - // definition. In case a serialized string is available - // to the instrumentation, the instrumentation SHOULD do the best effort to - // deserialize it to an array. When recorded on spans, it MAY be recorded as a - // JSON string if structured format is not supported and SHOULD be recorded in - // structured form otherwise. + // When the attribute is recorded on events, it MUST be recorded in structured + // form. When recorded on spans, it MAY be recorded as a JSON string if + // structured + // format is not supported and SHOULD be recorded in structured form otherwise. // // Since this attribute could be large, it's NOT RECOMMENDED to populate - // it by default. Instrumentations MAY provide a way to enable - // populating this attribute. + // non-required properties by default. Instrumentations MAY provide a way + // to enable populating optional properties. + // + // [Tool Definitions JSON Schema]: /docs/gen-ai/gen-ai-tool-definitions.json GenAIToolDefinitionsKey = attribute.Key("gen_ai.tool.definitions") // GenAIToolDescriptionKey is the attribute Key conforming to the @@ -6997,6 +7043,32 @@ const ( // // Examples: 180 GenAIUsageOutputTokensKey = attribute.Key("gen_ai.usage.output_tokens") + + // GenAIUsageReasoningOutputTokensKey is the attribute Key conforming to the + // "gen_ai.usage.reasoning.output_tokens" semantic conventions. It represents + // the number of output tokens used for reasoning (e.g. chain-of-thought, + // extended thinking). + // + // Type: int + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: 50 + // Note: The value SHOULD be included in `gen_ai.usage.output_tokens`. + GenAIUsageReasoningOutputTokensKey = attribute.Key("gen_ai.usage.reasoning.output_tokens") + + // GenAIWorkflowNameKey is the attribute Key conforming to the + // "gen_ai.workflow.name" semantic conventions. It represents the human-readable + // name of the GenAI workflow provided by the application. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "multi_agent_rag", "customer_support_pipeline" + // Note: This attribute can be populated in different frameworks eg: name of the + // first chain in LangChain OR name of the crew in CrewAI. + GenAIWorkflowNameKey = attribute.Key("gen_ai.workflow.name") ) // GenAIAgentDescription returns an attribute KeyValue conforming to the @@ -7139,6 +7211,13 @@ func GenAIRequestStopSequences(val ...string) attribute.KeyValue { return GenAIRequestStopSequencesKey.StringSlice(val) } +// GenAIRequestStream returns an attribute KeyValue conforming to the +// "gen_ai.request.stream" semantic conventions. It represents the indicates +// whether the GenAI request was made in streaming mode. +func GenAIRequestStream(val bool) attribute.KeyValue { + return GenAIRequestStreamKey.Bool(val) +} + // GenAIRequestTemperature returns an attribute KeyValue conforming to the // "gen_ai.request.temperature" semantic conventions. It represents the // temperature setting for the GenAI request. @@ -7182,6 +7261,15 @@ func GenAIResponseModel(val string) attribute.KeyValue { return GenAIResponseModelKey.String(val) } +// GenAIResponseTimeToFirstChunk returns an attribute KeyValue conforming to the +// "gen_ai.response.time_to_first_chunk" semantic conventions. It represents the +// time to first chunk in a streaming response, measured from request issuance, +// in seconds. The value is measured from when the client issues the generation +// request to when the first chunk is received in the response stream. +func GenAIResponseTimeToFirstChunk(val float64) attribute.KeyValue { + return GenAIResponseTimeToFirstChunkKey.Float64(val) +} + // GenAIRetrievalQueryText returns an attribute KeyValue conforming to the // "gen_ai.retrieval.query.text" semantic conventions. It represents the query // text used for retrieval. @@ -7245,6 +7333,21 @@ func GenAIUsageOutputTokens(val int) attribute.KeyValue { return GenAIUsageOutputTokensKey.Int(val) } +// GenAIUsageReasoningOutputTokens returns an attribute KeyValue conforming to +// the "gen_ai.usage.reasoning.output_tokens" semantic conventions. It represents +// the number of output tokens used for reasoning (e.g. chain-of-thought, +// extended thinking). +func GenAIUsageReasoningOutputTokens(val int) attribute.KeyValue { + return GenAIUsageReasoningOutputTokensKey.Int(val) +} + +// GenAIWorkflowName returns an attribute KeyValue conforming to the +// "gen_ai.workflow.name" semantic conventions. It represents the human-readable +// name of the GenAI workflow provided by the application. +func GenAIWorkflowName(val string) attribute.KeyValue { + return GenAIWorkflowNameKey.String(val) +} + // Enum values for gen_ai.operation.name var ( // Chat completion operation such as [OpenAI Chat API] @@ -7281,6 +7384,9 @@ var ( // Execute a tool // Stability: development GenAIOperationNameExecuteTool = GenAIOperationNameKey.String("execute_tool") + // Invoke GenAI workflow + // Stability: development + GenAIOperationNameInvokeWorkflow = GenAIOperationNameKey.String("invoke_workflow") ) // Enum values for gen_ai.output.type @@ -7335,7 +7441,7 @@ var ( // [Azure OpenAI] // Stability: development // - // [Azure OpenAI]: https://azure.microsoft.com/products/ai-services/openai-service/ + // [Azure OpenAI]: https://learn.microsoft.com/en-us/azure/ai-services/openai/overview GenAIProviderNameAzureAIOpenAI = GenAIProviderNameKey.String("azure.ai.openai") // [IBM Watsonx AI] // Stability: development @@ -7551,6 +7657,44 @@ var ( // Namespace: go const ( + // GoCPUDetailedStateKey is the attribute Key conforming to the + // "go.cpu.detailed_state" semantic conventions. It represents the detailed + // state of the CPU. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "gc/pause", "gc/mark/assist" + // Note: Value SHOULD match the specific CPU class reported by the Go runtime + // under `/cpu/classes/...`. The list of possible values is subject to change + // with the Go version used. + GoCPUDetailedStateKey = attribute.Key("go.cpu.detailed_state") + + // GoCPUStateKey is the attribute Key conforming to the "go.cpu.state" semantic + // conventions. It represents the state of the CPU. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "user", "gc" + GoCPUStateKey = attribute.Key("go.cpu.state") + + // GoMemoryDetailedTypeKey is the attribute Key conforming to the + // "go.memory.detailed_type" semantic conventions. It represents the detailed + // type of memory. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "heap/objects", "heap/free" + // Note: Value SHOULD match the specific memory class reported by the Go runtime + // under `/memory/classes/...`. The list of possible values is subject to change + // with the Go version used. + GoMemoryDetailedTypeKey = attribute.Key("go.memory.detailed_type") + // GoMemoryTypeKey is the attribute Key conforming to the "go.memory.type" // semantic conventions. It represents the type of memory. // @@ -7562,6 +7706,36 @@ const ( GoMemoryTypeKey = attribute.Key("go.memory.type") ) +// GoCPUDetailedState returns an attribute KeyValue conforming to the +// "go.cpu.detailed_state" semantic conventions. It represents the detailed state +// of the CPU. +func GoCPUDetailedState(val string) attribute.KeyValue { + return GoCPUDetailedStateKey.String(val) +} + +// GoMemoryDetailedType returns an attribute KeyValue conforming to the +// "go.memory.detailed_type" semantic conventions. It represents the detailed +// type of memory. +func GoMemoryDetailedType(val string) attribute.KeyValue { + return GoMemoryDetailedTypeKey.String(val) +} + +// Enum values for go.cpu.state +var ( + // CPU time spent running user Go code. + // Stability: development + GoCPUStateUser = GoCPUStateKey.String("user") + // CPU time spent performing garbage collection tasks. + // Stability: development + GoCPUStateGC = GoCPUStateKey.String("gc") + // CPU time spent returning unused memory to the underlying platform. + // Stability: development + GoCPUStateScavenge = GoCPUStateKey.String("scavenge") + // Available CPU time not spent executing any Go or Go runtime code. + // Stability: development + GoCPUStateIdle = GoCPUStateKey.String("idle") +) + // Enum values for go.memory.type var ( // Memory allocated from the heap that is reserved for stack space, whether or @@ -7584,7 +7758,8 @@ const ( // Stability: Development // // Examples: query findBookById { bookById(id: ?) { name } } - // Note: The value may be sanitized to exclude sensitive information. + // Note: If instrumentation can reliably identify and redact sensitive + // information it SHOULD do it. GraphQLDocumentKey = attribute.Key("graphql.document") // GraphQLOperationNameKey is the attribute Key conforming to the @@ -8335,7 +8510,7 @@ var ( const ( // HwBatteryCapacityKey is the attribute Key conforming to the // "hw.battery.capacity" semantic conventions. It represents the design capacity - // in Watts-hours or Amper-hours. + // in Watts-hours or Ampere-hours. // // Type: string // RequirementLevel: Recommended @@ -8637,7 +8812,7 @@ const ( // HwBatteryCapacity returns an attribute KeyValue conforming to the // "hw.battery.capacity" semantic conventions. It represents the design capacity -// in Watts-hours or Amper-hours. +// in Watts-hours or Ampere-hours. func HwBatteryCapacity(val string) attribute.KeyValue { return HwBatteryCapacityKey.String(val) } @@ -9026,7 +9201,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry-cluster" K8SClusterNameKey = attribute.Key("k8s.cluster.name") @@ -9037,7 +9212,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" // Note: K8s doesn't have support for obtaining a cluster ID. If this is ever @@ -9073,7 +9248,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "redis" K8SContainerNameKey = attribute.Key("k8s.container.name") @@ -9085,7 +9260,7 @@ const ( // // Type: int // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: K8SContainerRestartCountKey = attribute.Key("k8s.container.restart_count") @@ -9136,7 +9311,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SCronJobNameKey = attribute.Key("k8s.cronjob.name") @@ -9146,7 +9321,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SCronJobUIDKey = attribute.Key("k8s.cronjob.uid") @@ -9157,7 +9332,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SDaemonSetNameKey = attribute.Key("k8s.daemonset.name") @@ -9167,7 +9342,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SDaemonSetUIDKey = attribute.Key("k8s.daemonset.uid") @@ -9178,7 +9353,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SDeploymentNameKey = attribute.Key("k8s.deployment.name") @@ -9189,7 +9364,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SDeploymentUIDKey = attribute.Key("k8s.deployment.uid") @@ -9279,7 +9454,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SJobNameKey = attribute.Key("k8s.job.name") @@ -9289,7 +9464,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SJobUIDKey = attribute.Key("k8s.job.uid") @@ -9300,7 +9475,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "default" K8SNamespaceNameKey = attribute.Key("k8s.namespace.name") @@ -9365,27 +9540,128 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "node-1" K8SNodeNameKey = attribute.Key("k8s.node.name") + // K8SNodeSystemContainerNameKey is the attribute Key conforming to the + // "k8s.node.system_container.name" semantic conventions. It represents the name + // of the system container running on the K8s Node. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "kubelet", "runtime", "pods", "misc" + K8SNodeSystemContainerNameKey = attribute.Key("k8s.node.system_container.name") + // K8SNodeUIDKey is the attribute Key conforming to the "k8s.node.uid" semantic // conventions. It represents the UID of the Node. // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "1eb3a0c6-0477-4080-a9cb-0cb7db65c6a2" K8SNodeUIDKey = attribute.Key("k8s.node.uid") + // K8SPersistentvolumeNameKey is the attribute Key conforming to the + // "k8s.persistentvolume.name" semantic conventions. It represents the name of + // the PersistentVolume. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "pv-data-01" + K8SPersistentvolumeNameKey = attribute.Key("k8s.persistentvolume.name") + + // K8SPersistentvolumeReclaimPolicyKey is the attribute Key conforming to the + // "k8s.persistentvolume.reclaim_policy" semantic conventions. It represents the + // reclaim policy of the PersistentVolume. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Delete", "Retain", "Recycle" + // Note: This attribute aligns with the `persistentVolumeReclaimPolicy` field of + // the + // [K8s PersistentVolumeSpec]. + // + // [K8s PersistentVolumeSpec]: https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-v1/#PersistentVolumeSpec + K8SPersistentvolumeReclaimPolicyKey = attribute.Key("k8s.persistentvolume.reclaim_policy") + + // K8SPersistentvolumeStatusPhaseKey is the attribute Key conforming to the + // "k8s.persistentvolume.status.phase" semantic conventions. It represents the + // phase of the PersistentVolume. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Pending", "Available", "Bound", "Released", "Failed" + // Note: This attribute aligns with the `phase` field of the + // [K8s PersistentVolumeStatus]. + // + // [K8s PersistentVolumeStatus]: https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-v1/#PersistentVolumeStatus + K8SPersistentvolumeStatusPhaseKey = attribute.Key("k8s.persistentvolume.status.phase") + + // K8SPersistentvolumeUIDKey is the attribute Key conforming to the + // "k8s.persistentvolume.uid" semantic conventions. It represents the UID of the + // PersistentVolume. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" + K8SPersistentvolumeUIDKey = attribute.Key("k8s.persistentvolume.uid") + + // K8SPersistentvolumeclaimNameKey is the attribute Key conforming to the + // "k8s.persistentvolumeclaim.name" semantic conventions. It represents the name + // of the PersistentVolumeClaim. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "pvc-data-01" + K8SPersistentvolumeclaimNameKey = attribute.Key("k8s.persistentvolumeclaim.name") + + // K8SPersistentvolumeclaimStatusPhaseKey is the attribute Key conforming to the + // "k8s.persistentvolumeclaim.status.phase" semantic conventions. It represents + // the phase of the PersistentVolumeClaim. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Pending", "Bound", "Lost" + // Note: This attribute aligns with the `phase` field of the + // [K8s PersistentVolumeClaimStatus]. + // + // [K8s PersistentVolumeClaimStatus]: https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-claim-v1/#PersistentVolumeClaimStatus + K8SPersistentvolumeclaimStatusPhaseKey = attribute.Key("k8s.persistentvolumeclaim.status.phase") + + // K8SPersistentvolumeclaimUIDKey is the attribute Key conforming to the + // "k8s.persistentvolumeclaim.uid" semantic conventions. It represents the UID + // of the PersistentVolumeClaim. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" + K8SPersistentvolumeclaimUIDKey = attribute.Key("k8s.persistentvolumeclaim.uid") + // K8SPodHostnameKey is the attribute Key conforming to the "k8s.pod.hostname" // semantic conventions. It represents the specifies the hostname of the Pod. // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "collector-gateway" // Note: The K8s Pod spec has an optional hostname field, which can be used to @@ -9405,7 +9681,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "172.18.0.2" // Note: This attribute aligns with the `podIP` field of the @@ -9419,7 +9695,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry-pod-autoconf" K8SPodNameKey = attribute.Key("k8s.pod.name") @@ -9430,7 +9706,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "2025-12-04T08:41:03Z" // Note: Date and time at which the object was acknowledged by the Kubelet. @@ -9474,7 +9750,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SPodUIDKey = attribute.Key("k8s.pod.uid") @@ -9485,7 +9761,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SReplicaSetNameKey = attribute.Key("k8s.replicaset.name") @@ -9496,7 +9772,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SReplicaSetUIDKey = attribute.Key("k8s.replicaset.uid") @@ -9709,7 +9985,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "opentelemetry" K8SStatefulSetNameKey = attribute.Key("k8s.statefulset.name") @@ -9720,7 +9996,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Beta + // Stability: Release_Candidate // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SStatefulSetUIDKey = attribute.Key("k8s.statefulset.uid") @@ -10005,12 +10281,80 @@ func K8SNodeName(val string) attribute.KeyValue { return K8SNodeNameKey.String(val) } +// K8SNodeSystemContainerName returns an attribute KeyValue conforming to the +// "k8s.node.system_container.name" semantic conventions. It represents the name +// of the system container running on the K8s Node. +func K8SNodeSystemContainerName(val string) attribute.KeyValue { + return K8SNodeSystemContainerNameKey.String(val) +} + // K8SNodeUID returns an attribute KeyValue conforming to the "k8s.node.uid" // semantic conventions. It represents the UID of the Node. func K8SNodeUID(val string) attribute.KeyValue { return K8SNodeUIDKey.String(val) } +// K8SPersistentvolumeAnnotation returns an attribute KeyValue conforming to the +// "k8s.persistentvolume.annotation" semantic conventions. It represents the +// annotation placed on the PersistentVolume, the `` being the annotation +// name, the value being the annotation value, even if the value is empty. +func K8SPersistentvolumeAnnotation(key string, val string) attribute.KeyValue { + return attribute.String("k8s.persistentvolume.annotation."+key, val) +} + +// K8SPersistentvolumeLabel returns an attribute KeyValue conforming to the +// "k8s.persistentvolume.label" semantic conventions. It represents the label +// placed on the PersistentVolume, the `` being the label name, the value +// being the label value, even if the value is empty. +func K8SPersistentvolumeLabel(key string, val string) attribute.KeyValue { + return attribute.String("k8s.persistentvolume.label."+key, val) +} + +// K8SPersistentvolumeName returns an attribute KeyValue conforming to the +// "k8s.persistentvolume.name" semantic conventions. It represents the name of +// the PersistentVolume. +func K8SPersistentvolumeName(val string) attribute.KeyValue { + return K8SPersistentvolumeNameKey.String(val) +} + +// K8SPersistentvolumeUID returns an attribute KeyValue conforming to the +// "k8s.persistentvolume.uid" semantic conventions. It represents the UID of the +// PersistentVolume. +func K8SPersistentvolumeUID(val string) attribute.KeyValue { + return K8SPersistentvolumeUIDKey.String(val) +} + +// K8SPersistentvolumeclaimAnnotation returns an attribute KeyValue conforming to +// the "k8s.persistentvolumeclaim.annotation" semantic conventions. It represents +// the annotation placed on the PersistentVolumeClaim, the `` being the +// annotation name, the value being the annotation value, even if the value is +// empty. +func K8SPersistentvolumeclaimAnnotation(key string, val string) attribute.KeyValue { + return attribute.String("k8s.persistentvolumeclaim.annotation."+key, val) +} + +// K8SPersistentvolumeclaimLabel returns an attribute KeyValue conforming to the +// "k8s.persistentvolumeclaim.label" semantic conventions. It represents the +// label placed on the PersistentVolumeClaim, the `` being the label name, +// the value being the label value, even if the value is empty. +func K8SPersistentvolumeclaimLabel(key string, val string) attribute.KeyValue { + return attribute.String("k8s.persistentvolumeclaim.label."+key, val) +} + +// K8SPersistentvolumeclaimName returns an attribute KeyValue conforming to the +// "k8s.persistentvolumeclaim.name" semantic conventions. It represents the name +// of the PersistentVolumeClaim. +func K8SPersistentvolumeclaimName(val string) attribute.KeyValue { + return K8SPersistentvolumeclaimNameKey.String(val) +} + +// K8SPersistentvolumeclaimUID returns an attribute KeyValue conforming to the +// "k8s.persistentvolumeclaim.uid" semantic conventions. It represents the UID of +// the PersistentVolumeClaim. +func K8SPersistentvolumeclaimUID(val string) attribute.KeyValue { + return K8SPersistentvolumeclaimUIDKey.String(val) +} + // K8SPodAnnotation returns an attribute KeyValue conforming to the // "k8s.pod.annotation" semantic conventions. It represents the annotation placed // on the Pod, the `` being the annotation name, the value being the @@ -10318,6 +10662,51 @@ var ( K8SNodeConditionTypeNetworkUnavailable = K8SNodeConditionTypeKey.String("NetworkUnavailable") ) +// Enum values for k8s.persistentvolume.reclaim_policy +var ( + // The volume will be deleted when released from its claim. + // Stability: development + K8SPersistentvolumeReclaimPolicyDelete = K8SPersistentvolumeReclaimPolicyKey.String("Delete") + // The volume will be recycled (basic scrub) when released from its claim. + // Stability: development + K8SPersistentvolumeReclaimPolicyRecycle = K8SPersistentvolumeReclaimPolicyKey.String("Recycle") + // The volume will be retained when released from its claim. + // Stability: development + K8SPersistentvolumeReclaimPolicyRetain = K8SPersistentvolumeReclaimPolicyKey.String("Retain") +) + +// Enum values for k8s.persistentvolume.status.phase +var ( + // The volume is available and not yet bound to a claim. + // Stability: development + K8SPersistentvolumeStatusPhaseAvailable = K8SPersistentvolumeStatusPhaseKey.String("Available") + // The volume is bound to a claim. + // Stability: development + K8SPersistentvolumeStatusPhaseBound = K8SPersistentvolumeStatusPhaseKey.String("Bound") + // The volume has failed its automatic reclamation. + // Stability: development + K8SPersistentvolumeStatusPhaseFailed = K8SPersistentvolumeStatusPhaseKey.String("Failed") + // The volume is being provisioned. + // Stability: development + K8SPersistentvolumeStatusPhasePending = K8SPersistentvolumeStatusPhaseKey.String("Pending") + // The claim has been deleted but the volume is not yet available. + // Stability: development + K8SPersistentvolumeStatusPhaseReleased = K8SPersistentvolumeStatusPhaseKey.String("Released") +) + +// Enum values for k8s.persistentvolumeclaim.status.phase +var ( + // The claim is bound to a volume. + // Stability: development + K8SPersistentvolumeclaimStatusPhaseBound = K8SPersistentvolumeclaimStatusPhaseKey.String("Bound") + // The claim has lost its underlying volume (the volume does not exist anymore). + // Stability: development + K8SPersistentvolumeclaimStatusPhaseLost = K8SPersistentvolumeclaimStatusPhaseKey.String("Lost") + // The claim has not yet been bound to a volume. + // Stability: development + K8SPersistentvolumeclaimStatusPhasePending = K8SPersistentvolumeclaimStatusPhaseKey.String("Pending") +) + // Enum values for k8s.pod.status.phase var ( // The pod has been accepted by the system, but one or more of the containers @@ -12669,7 +13058,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Stable // // Examples: "browser.mouse.click", "device.app.lifecycle" // Note: This attribute SHOULD be used by non-OTLP exporters when destination @@ -13240,14 +13629,27 @@ const ( // ProcessExecutableBuildIDHtlhashKey is the attribute Key conforming to the // "process.executable.build_id.htlhash" semantic conventions. It represents the - // profiling specific build ID for executables. See the OTel specification for - // Profiles for more information. + // deterministic build ID for executables. // // Type: string // RequirementLevel: Recommended // Stability: Development // // Examples: "600DCAFE4A110000F2BF38C493F5FB92" + // Note: GNU and Go build IDs may be stripped or unavailable in some + // environments + // (e.g., Alpine Linux, Docker images). This attribute provides a deterministic + // build ID computed by hashing the first and last 4096 bytes of the file + // along with its length: + // + // ``` + // Input ← Concat(File[:4096], File[-4096:], BigEndianUInt64(Len(File))) + // Digest ← SHA256(Input) + // BuildID ← Digest[:16] + // ``` + // + // The result is the first 16 bytes (128 bits) of the SHA256 digest, + // represented as a hex string. ProcessExecutableBuildIDHtlhashKey = attribute.Key("process.executable.build_id.htlhash") // ProcessExecutableNameKey is the attribute Key conforming to the @@ -13603,8 +14005,7 @@ func ProcessExecutableBuildIDGo(val string) attribute.KeyValue { // ProcessExecutableBuildIDHtlhash returns an attribute KeyValue conforming to // the "process.executable.build_id.htlhash" semantic conventions. It represents -// the profiling specific build ID for executables. See the OTel specification -// for Profiles for more information. +// the deterministic build ID for executables. func ProcessExecutableBuildIDHtlhash(val string) attribute.KeyValue { return ProcessExecutableBuildIDHtlhashKey.String(val) } @@ -14317,9 +14718,11 @@ const ( // Examples: "shoppingcart" // Note: MUST be the same for all instances of horizontally scaled services. If // the value was not specified, SDKs MUST fallback to `unknown_service:` - // concatenated with [`process.executable.name`], e.g. `unknown_service:bash`. - // If `process.executable.name` is not available, the value MUST be set to + // concatenated with the process executable name, e.g. `unknown_service:bash`. + // If the process executable name is not available, the value MUST be set to // `unknown_service`. + // The process executable name is the name of the process executable, the same + // value as described by the [`process.executable.name`] resource attribute. // // [`process.executable.name`]: process.md ServiceNameKey = attribute.Key("service.name") @@ -14643,6 +15046,17 @@ const ( // Examples: "ext4" SystemFilesystemTypeKey = attribute.Key("system.filesystem.type") + // SystemMemoryLinuxHugepagesStateKey is the attribute Key conforming to the + // "system.memory.linux.hugepages.state" semantic conventions. It represents the + // Linux HugePages memory state. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "free", "used" + SystemMemoryLinuxHugepagesStateKey = attribute.Key("system.memory.linux.hugepages.state") + // SystemMemoryLinuxSlabStateKey is the attribute Key conforming to the // "system.memory.linux.slab.state" semantic conventions. It represents the // Linux Slab memory state. @@ -14753,6 +15167,16 @@ var ( SystemFilesystemTypeExt4 = SystemFilesystemTypeKey.String("ext4") ) +// Enum values for system.memory.linux.hugepages.state +var ( + // free + // Stability: development + SystemMemoryLinuxHugepagesStateFree = SystemMemoryLinuxHugepagesStateKey.String("free") + // used + // Stability: development + SystemMemoryLinuxHugepagesStateUsed = SystemMemoryLinuxHugepagesStateKey.String("used") +) + // Enum values for system.memory.linux.slab.state var ( // reclaimable @@ -14817,7 +15241,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Stable // // Examples: "parts-unlimited-java" // Note: Official auto instrumentation agents and distributions SHOULD set the @@ -14832,7 +15256,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Stable // // Examples: "1.2.3" TelemetryDistroVersionKey = attribute.Key("telemetry.distro.version") diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/doc.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/doc.go similarity index 82% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/doc.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/doc.go index c5c41e4d2..a45d424d8 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/doc.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/doc.go @@ -1,9 +1,11 @@ +// Code generated from semantic convention specification. DO NOT EDIT. + // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 // Package semconv implements OpenTelemetry semantic conventions. // // OpenTelemetry semantic conventions are agreed standardized naming -// patterns for OpenTelemetry things. This package represents the v1.40.0 +// patterns for OpenTelemetry things. This package represents the v1.41.0 // version of the OpenTelemetry semantic conventions. -package semconv // import "go.opentelemetry.io/otel/semconv/v1.40.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.41.0" diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/error_type.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/error_type.go similarity index 75% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/error_type.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/error_type.go index 6d26e5282..0b13f0de8 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/error_type.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/error_type.go @@ -1,10 +1,13 @@ +// Code generated from semantic convention specification. DO NOT EDIT. + // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.40.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.41.0" import ( "errors" + "fmt" "reflect" "go.opentelemetry.io/otel/attribute" @@ -22,7 +25,8 @@ import ( // the returned attribute has that method's return value. If multiple errors in // the chain implement this method, the value from the first match found by // [errors.As] is used. Otherwise, the returned attribute has a value derived -// from the concrete type of err. +// from the concrete type of err after unwrapping any wrappers created with +// [fmt.Errorf]. // // The key of the returned attribute is [ErrorTypeKey]. func ErrorType(err error) attribute.KeyValue { @@ -50,7 +54,7 @@ func errorType(err error) string { // Fallback to reflection if the ErrorType method is not supported or // returns an empty value. - t := reflect.TypeOf(err) + t := reflect.TypeOf(unwrapFmtWrapped(err)) pkg, name := t.PkgPath(), t.Name() if pkg != "" && name != "" { s = pkg + "." + name @@ -64,3 +68,16 @@ func errorType(err error) string { } return s } + +var fmtWrapErrorType = reflect.TypeOf(fmt.Errorf("wrapped: %w", errors.New("err"))) + +func unwrapFmtWrapped(err error) error { + for reflect.TypeOf(err) == fmtWrapErrorType { + u := errors.Unwrap(err) + if u == nil { + return err // When the wrapped error is nil, use the concrete type of the wrapper. + } + err = u + } + return err +} diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/exception.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/exception.go similarity index 77% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/exception.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/exception.go index 6a26231a1..5f0151aff 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/exception.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/exception.go @@ -1,7 +1,9 @@ +// Code generated from semantic convention specification. DO NOT EDIT. + // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.40.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.41.0" const ( // ExceptionEventName is the name of the Span event representing an exception. diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/otelconv/metric.go similarity index 66% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/otelconv/metric.go index 901da8698..d50e19849 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/otelconv/metric.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/otelconv/metric.go @@ -197,6 +197,9 @@ func (m SDKExporterLogExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -204,6 +207,7 @@ func (m SDKExporterLogExported) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -228,6 +232,9 @@ func (m SDKExporterLogExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterLogExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -235,6 +242,7 @@ func (m SDKExporterLogExported) AddSet(ctx context.Context, incr int64, set attr o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -277,6 +285,100 @@ func (SDKExporterLogExported) AttrServerPort(val int) attribute.KeyValue { return attribute.Int("server.port", val) } +// SDKExporterLogExportedObservable is an instrument used to record metric values +// conforming to the "otel.sdk.exporter.log.exported" semantic conventions. It +// represents the number of log records for which the export has finished, either +// successful or failed. +type SDKExporterLogExportedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKExporterLogExportedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of log records for which the export has finished, either successful or failed."), + metric.WithUnit("{log_record}"), +} + +// NewSDKExporterLogExportedObservable returns a new +// SDKExporterLogExportedObservable instrument. +func NewSDKExporterLogExportedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKExporterLogExportedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterLogExportedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterLogExportedObservableOpts + } else { + opt = append(opt, newSDKExporterLogExportedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.exporter.log.exported", + opt..., + ) + if err != nil { + return SDKExporterLogExportedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKExporterLogExportedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterLogExportedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterLogExportedObservable) Name() string { + return "otel.sdk.exporter.log.exported" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterLogExportedObservable) Unit() string { + return "{log_record}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterLogExportedObservable) Description() string { + return "The number of log records for which the export has finished, either successful or failed." +} + +// AttrErrorType returns an optional attribute for the "error.type" semantic +// convention. It represents the describes a class of error the operation ended +// with. +func (SDKExporterLogExportedObservable) AttrErrorType(val ErrorTypeAttr) attribute.KeyValue { + return attribute.String("error.type", string(val)) +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterLogExportedObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterLogExportedObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterLogExportedObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterLogExportedObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKExporterLogInflight is an instrument used to record metric values // conforming to the "otel.sdk.exporter.log.inflight" semantic conventions. It // represents the number of log records which were passed to the exporter, but @@ -347,6 +449,9 @@ func (m SDKExporterLogInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -354,6 +459,7 @@ func (m SDKExporterLogInflight) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -373,6 +479,9 @@ func (m SDKExporterLogInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterLogInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -380,6 +489,7 @@ func (m SDKExporterLogInflight) AddSet(ctx context.Context, incr int64, set attr o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -415,6 +525,93 @@ func (SDKExporterLogInflight) AttrServerPort(val int) attribute.KeyValue { return attribute.Int("server.port", val) } +// SDKExporterLogInflightObservable is an instrument used to record metric values +// conforming to the "otel.sdk.exporter.log.inflight" semantic conventions. It +// represents the number of log records which were passed to the exporter, but +// that have not been exported yet (neither successful, nor failed). +type SDKExporterLogInflightObservable struct { + metric.Int64ObservableUpDownCounter +} + +var newSDKExporterLogInflightObservableOpts = []metric.Int64ObservableUpDownCounterOption{ + metric.WithDescription("The number of log records which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)."), + metric.WithUnit("{log_record}"), +} + +// NewSDKExporterLogInflightObservable returns a new +// SDKExporterLogInflightObservable instrument. +func NewSDKExporterLogInflightObservable( + m metric.Meter, + opt ...metric.Int64ObservableUpDownCounterOption, +) (SDKExporterLogInflightObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterLogInflightObservable{noop.Int64ObservableUpDownCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterLogInflightObservableOpts + } else { + opt = append(opt, newSDKExporterLogInflightObservableOpts...) + } + + i, err := m.Int64ObservableUpDownCounter( + "otel.sdk.exporter.log.inflight", + opt..., + ) + if err != nil { + return SDKExporterLogInflightObservable{noop.Int64ObservableUpDownCounter{}}, err + } + return SDKExporterLogInflightObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterLogInflightObservable) Inst() metric.Int64ObservableUpDownCounter { + return m.Int64ObservableUpDownCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterLogInflightObservable) Name() string { + return "otel.sdk.exporter.log.inflight" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterLogInflightObservable) Unit() string { + return "{log_record}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterLogInflightObservable) Description() string { + return "The number of log records which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)." +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterLogInflightObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterLogInflightObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterLogInflightObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterLogInflightObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKExporterMetricDataPointExported is an instrument used to record metric // values conforming to the "otel.sdk.exporter.metric_data_point.exported" // semantic conventions. It represents the number of metric data points for which @@ -491,6 +688,9 @@ func (m SDKExporterMetricDataPointExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -498,6 +698,7 @@ func (m SDKExporterMetricDataPointExported) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -522,6 +723,9 @@ func (m SDKExporterMetricDataPointExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterMetricDataPointExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -529,6 +733,7 @@ func (m SDKExporterMetricDataPointExported) AddSet(ctx context.Context, incr int o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -571,6 +776,100 @@ func (SDKExporterMetricDataPointExported) AttrServerPort(val int) attribute.KeyV return attribute.Int("server.port", val) } +// SDKExporterMetricDataPointExportedObservable is an instrument used to record +// metric values conforming to the "otel.sdk.exporter.metric_data_point.exported" +// semantic conventions. It represents the number of metric data points for which +// the export has finished, either successful or failed. +type SDKExporterMetricDataPointExportedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKExporterMetricDataPointExportedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of metric data points for which the export has finished, either successful or failed."), + metric.WithUnit("{data_point}"), +} + +// NewSDKExporterMetricDataPointExportedObservable returns a new +// SDKExporterMetricDataPointExportedObservable instrument. +func NewSDKExporterMetricDataPointExportedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKExporterMetricDataPointExportedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterMetricDataPointExportedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterMetricDataPointExportedObservableOpts + } else { + opt = append(opt, newSDKExporterMetricDataPointExportedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.exporter.metric_data_point.exported", + opt..., + ) + if err != nil { + return SDKExporterMetricDataPointExportedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKExporterMetricDataPointExportedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterMetricDataPointExportedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterMetricDataPointExportedObservable) Name() string { + return "otel.sdk.exporter.metric_data_point.exported" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterMetricDataPointExportedObservable) Unit() string { + return "{data_point}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterMetricDataPointExportedObservable) Description() string { + return "The number of metric data points for which the export has finished, either successful or failed." +} + +// AttrErrorType returns an optional attribute for the "error.type" semantic +// convention. It represents the describes a class of error the operation ended +// with. +func (SDKExporterMetricDataPointExportedObservable) AttrErrorType(val ErrorTypeAttr) attribute.KeyValue { + return attribute.String("error.type", string(val)) +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterMetricDataPointExportedObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterMetricDataPointExportedObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterMetricDataPointExportedObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterMetricDataPointExportedObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKExporterMetricDataPointInflight is an instrument used to record metric // values conforming to the "otel.sdk.exporter.metric_data_point.inflight" // semantic conventions. It represents the number of metric data points which @@ -643,6 +942,9 @@ func (m SDKExporterMetricDataPointInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -650,6 +952,7 @@ func (m SDKExporterMetricDataPointInflight) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -669,6 +972,9 @@ func (m SDKExporterMetricDataPointInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterMetricDataPointInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -676,6 +982,7 @@ func (m SDKExporterMetricDataPointInflight) AddSet(ctx context.Context, incr int o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -711,6 +1018,94 @@ func (SDKExporterMetricDataPointInflight) AttrServerPort(val int) attribute.KeyV return attribute.Int("server.port", val) } +// SDKExporterMetricDataPointInflightObservable is an instrument used to record +// metric values conforming to the "otel.sdk.exporter.metric_data_point.inflight" +// semantic conventions. It represents the number of metric data points which +// were passed to the exporter, but that have not been exported yet (neither +// successful, nor failed). +type SDKExporterMetricDataPointInflightObservable struct { + metric.Int64ObservableUpDownCounter +} + +var newSDKExporterMetricDataPointInflightObservableOpts = []metric.Int64ObservableUpDownCounterOption{ + metric.WithDescription("The number of metric data points which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)."), + metric.WithUnit("{data_point}"), +} + +// NewSDKExporterMetricDataPointInflightObservable returns a new +// SDKExporterMetricDataPointInflightObservable instrument. +func NewSDKExporterMetricDataPointInflightObservable( + m metric.Meter, + opt ...metric.Int64ObservableUpDownCounterOption, +) (SDKExporterMetricDataPointInflightObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterMetricDataPointInflightObservable{noop.Int64ObservableUpDownCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterMetricDataPointInflightObservableOpts + } else { + opt = append(opt, newSDKExporterMetricDataPointInflightObservableOpts...) + } + + i, err := m.Int64ObservableUpDownCounter( + "otel.sdk.exporter.metric_data_point.inflight", + opt..., + ) + if err != nil { + return SDKExporterMetricDataPointInflightObservable{noop.Int64ObservableUpDownCounter{}}, err + } + return SDKExporterMetricDataPointInflightObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterMetricDataPointInflightObservable) Inst() metric.Int64ObservableUpDownCounter { + return m.Int64ObservableUpDownCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterMetricDataPointInflightObservable) Name() string { + return "otel.sdk.exporter.metric_data_point.inflight" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterMetricDataPointInflightObservable) Unit() string { + return "{data_point}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterMetricDataPointInflightObservable) Description() string { + return "The number of metric data points which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)." +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterMetricDataPointInflightObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterMetricDataPointInflightObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterMetricDataPointInflightObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterMetricDataPointInflightObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKExporterOperationDuration is an instrument used to record metric values // conforming to the "otel.sdk.exporter.operation.duration" semantic conventions. // It represents the duration of exporting a batch of telemetry records. @@ -788,6 +1183,9 @@ func (m SDKExporterOperationDuration) Record( val float64, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val) return @@ -795,6 +1193,7 @@ func (m SDKExporterOperationDuration) Record( o := recOptPool.Get().(*[]metric.RecordOption) defer func() { + clear(*o) *o = (*o)[:0] recOptPool.Put(o) }() @@ -821,6 +1220,9 @@ func (m SDKExporterOperationDuration) Record( // [http]: https://github.com/open-telemetry/opentelemetry-proto/blob/v1.5.0/docs/specification.md#full-success-1 // [grpc]: https://github.com/open-telemetry/opentelemetry-proto/blob/v1.5.0/docs/specification.md#full-success func (m SDKExporterOperationDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -828,6 +1230,7 @@ func (m SDKExporterOperationDuration) RecordSet(ctx context.Context, val float64 o := recOptPool.Get().(*[]metric.RecordOption) defer func() { + clear(*o) *o = (*o)[:0] recOptPool.Put(o) }() @@ -959,6 +1362,9 @@ func (m SDKExporterSpanExported) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -966,6 +1372,7 @@ func (m SDKExporterSpanExported) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -990,6 +1397,9 @@ func (m SDKExporterSpanExported) Add( // If no rejection reason is available, `rejected` SHOULD be used as value for // `error.type`. func (m SDKExporterSpanExported) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -997,6 +1407,7 @@ func (m SDKExporterSpanExported) AddSet(ctx context.Context, incr int64, set att o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1039,6 +1450,100 @@ func (SDKExporterSpanExported) AttrServerPort(val int) attribute.KeyValue { return attribute.Int("server.port", val) } +// SDKExporterSpanExportedObservable is an instrument used to record metric +// values conforming to the "otel.sdk.exporter.span.exported" semantic +// conventions. It represents the number of spans for which the export has +// finished, either successful or failed. +type SDKExporterSpanExportedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKExporterSpanExportedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of spans for which the export has finished, either successful or failed."), + metric.WithUnit("{span}"), +} + +// NewSDKExporterSpanExportedObservable returns a new +// SDKExporterSpanExportedObservable instrument. +func NewSDKExporterSpanExportedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKExporterSpanExportedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterSpanExportedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterSpanExportedObservableOpts + } else { + opt = append(opt, newSDKExporterSpanExportedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.exporter.span.exported", + opt..., + ) + if err != nil { + return SDKExporterSpanExportedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKExporterSpanExportedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterSpanExportedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterSpanExportedObservable) Name() string { + return "otel.sdk.exporter.span.exported" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterSpanExportedObservable) Unit() string { + return "{span}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterSpanExportedObservable) Description() string { + return "The number of spans for which the export has finished, either successful or failed." +} + +// AttrErrorType returns an optional attribute for the "error.type" semantic +// convention. It represents the describes a class of error the operation ended +// with. +func (SDKExporterSpanExportedObservable) AttrErrorType(val ErrorTypeAttr) attribute.KeyValue { + return attribute.String("error.type", string(val)) +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterSpanExportedObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterSpanExportedObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterSpanExportedObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterSpanExportedObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKExporterSpanInflight is an instrument used to record metric values // conforming to the "otel.sdk.exporter.span.inflight" semantic conventions. It // represents the number of spans which were passed to the exporter, but that @@ -1109,6 +1614,9 @@ func (m SDKExporterSpanInflight) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -1116,6 +1624,7 @@ func (m SDKExporterSpanInflight) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1135,6 +1644,9 @@ func (m SDKExporterSpanInflight) Add( // For successful exports, `error.type` MUST NOT be set. For failed exports, // `error.type` MUST contain the failure cause. func (m SDKExporterSpanInflight) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -1142,6 +1654,7 @@ func (m SDKExporterSpanInflight) AddSet(ctx context.Context, incr int64, set att o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1177,6 +1690,94 @@ func (SDKExporterSpanInflight) AttrServerPort(val int) attribute.KeyValue { return attribute.Int("server.port", val) } +// SDKExporterSpanInflightObservable is an instrument used to record metric +// values conforming to the "otel.sdk.exporter.span.inflight" semantic +// conventions. It represents the number of spans which were passed to the +// exporter, but that have not been exported yet (neither successful, nor +// failed). +type SDKExporterSpanInflightObservable struct { + metric.Int64ObservableUpDownCounter +} + +var newSDKExporterSpanInflightObservableOpts = []metric.Int64ObservableUpDownCounterOption{ + metric.WithDescription("The number of spans which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)."), + metric.WithUnit("{span}"), +} + +// NewSDKExporterSpanInflightObservable returns a new +// SDKExporterSpanInflightObservable instrument. +func NewSDKExporterSpanInflightObservable( + m metric.Meter, + opt ...metric.Int64ObservableUpDownCounterOption, +) (SDKExporterSpanInflightObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKExporterSpanInflightObservable{noop.Int64ObservableUpDownCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKExporterSpanInflightObservableOpts + } else { + opt = append(opt, newSDKExporterSpanInflightObservableOpts...) + } + + i, err := m.Int64ObservableUpDownCounter( + "otel.sdk.exporter.span.inflight", + opt..., + ) + if err != nil { + return SDKExporterSpanInflightObservable{noop.Int64ObservableUpDownCounter{}}, err + } + return SDKExporterSpanInflightObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKExporterSpanInflightObservable) Inst() metric.Int64ObservableUpDownCounter { + return m.Int64ObservableUpDownCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKExporterSpanInflightObservable) Name() string { + return "otel.sdk.exporter.span.inflight" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKExporterSpanInflightObservable) Unit() string { + return "{span}" +} + +// Description returns the semantic convention description of the instrument +func (SDKExporterSpanInflightObservable) Description() string { + return "The number of spans which were passed to the exporter, but that have not been exported yet (neither successful, nor failed)." +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKExporterSpanInflightObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKExporterSpanInflightObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + +// AttrServerAddress returns an optional attribute for the "server.address" +// semantic convention. It represents the server domain name if available without +// reverse DNS lookup; otherwise, IP address or Unix domain socket name. +func (SDKExporterSpanInflightObservable) AttrServerAddress(val string) attribute.KeyValue { + return attribute.String("server.address", val) +} + +// AttrServerPort returns an optional attribute for the "server.port" semantic +// convention. It represents the server port number. +func (SDKExporterSpanInflightObservable) AttrServerPort(val int) attribute.KeyValue { + return attribute.Int("server.port", val) +} + // SDKLogCreated is an instrument used to record metric values conforming to the // "otel.sdk.log.created" semantic conventions. It represents the number of logs // submitted to enabled SDK Loggers. @@ -1237,6 +1838,9 @@ func (SDKLogCreated) Description() string { // Add adds incr to the existing count for attrs. func (m SDKLogCreated) Add(ctx context.Context, incr int64, attrs ...attribute.KeyValue) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1244,29 +1848,92 @@ func (m SDKLogCreated) Add(ctx context.Context, incr int64, attrs ...attribute.K o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() - *o = append(*o, metric.WithAttributes(attrs...)) - m.Int64Counter.Add(ctx, incr, *o...) + *o = append(*o, metric.WithAttributes(attrs...)) + m.Int64Counter.Add(ctx, incr, *o...) +} + +// AddSet adds incr to the existing count for set. +func (m SDKLogCreated) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } + if set.Len() == 0 { + m.Int64Counter.Add(ctx, incr) + return + } + + o := addOptPool.Get().(*[]metric.AddOption) + defer func() { + clear(*o) + *o = (*o)[:0] + addOptPool.Put(o) + }() + + *o = append(*o, metric.WithAttributeSet(set)) + m.Int64Counter.Add(ctx, incr, *o...) +} + +// SDKLogCreatedObservable is an instrument used to record metric values +// conforming to the "otel.sdk.log.created" semantic conventions. It represents +// the number of logs submitted to enabled SDK Loggers. +type SDKLogCreatedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKLogCreatedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of logs submitted to enabled SDK Loggers."), + metric.WithUnit("{log_record}"), +} + +// NewSDKLogCreatedObservable returns a new SDKLogCreatedObservable instrument. +func NewSDKLogCreatedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKLogCreatedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKLogCreatedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKLogCreatedObservableOpts + } else { + opt = append(opt, newSDKLogCreatedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.log.created", + opt..., + ) + if err != nil { + return SDKLogCreatedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKLogCreatedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKLogCreatedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter } -// AddSet adds incr to the existing count for set. -func (m SDKLogCreated) AddSet(ctx context.Context, incr int64, set attribute.Set) { - if set.Len() == 0 { - m.Int64Counter.Add(ctx, incr) - return - } +// Name returns the semantic convention name of the instrument. +func (SDKLogCreatedObservable) Name() string { + return "otel.sdk.log.created" +} - o := addOptPool.Get().(*[]metric.AddOption) - defer func() { - *o = (*o)[:0] - addOptPool.Put(o) - }() +// Unit returns the semantic convention unit of the instrument +func (SDKLogCreatedObservable) Unit() string { + return "{log_record}" +} - *o = append(*o, metric.WithAttributeSet(set)) - m.Int64Counter.Add(ctx, incr, *o...) +// Description returns the semantic convention description of the instrument +func (SDKLogCreatedObservable) Description() string { + return "The number of logs submitted to enabled SDK Loggers." } // SDKMetricReaderCollectionDuration is an instrument used to record metric @@ -1343,6 +2010,9 @@ func (m SDKMetricReaderCollectionDuration) Record( val float64, attrs ...attribute.KeyValue, ) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Float64Histogram.Record(ctx, val) return @@ -1350,6 +2020,7 @@ func (m SDKMetricReaderCollectionDuration) Record( o := recOptPool.Get().(*[]metric.RecordOption) defer func() { + clear(*o) *o = (*o)[:0] recOptPool.Put(o) }() @@ -1372,6 +2043,9 @@ func (m SDKMetricReaderCollectionDuration) Record( // while others fail. In that case `error.type` SHOULD be set to any of the // failure causes. func (m SDKMetricReaderCollectionDuration) RecordSet(ctx context.Context, val float64, set attribute.Set) { + if !m.Float64Histogram.Enabled(ctx) { + return + } if set.Len() == 0 { m.Float64Histogram.Record(ctx, val) return @@ -1379,6 +2053,7 @@ func (m SDKMetricReaderCollectionDuration) RecordSet(ctx context.Context, val fl o := recOptPool.Get().(*[]metric.RecordOption) defer func() { + clear(*o) *o = (*o)[:0] recOptPool.Put(o) }() @@ -1481,6 +2156,9 @@ func (m SDKProcessorLogProcessed) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1488,6 +2166,7 @@ func (m SDKProcessorLogProcessed) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1510,6 +2189,9 @@ func (m SDKProcessorLogProcessed) Add( // considered to be processed already when it has been submitted to the exporter, // not when the corresponding export call has finished. func (m SDKProcessorLogProcessed) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1517,6 +2199,7 @@ func (m SDKProcessorLogProcessed) AddSet(ctx context.Context, incr int64, set at o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1547,6 +2230,88 @@ func (SDKProcessorLogProcessed) AttrComponentType(val ComponentTypeAttr) attribu return attribute.String("otel.component.type", string(val)) } +// SDKProcessorLogProcessedObservable is an instrument used to record metric +// values conforming to the "otel.sdk.processor.log.processed" semantic +// conventions. It represents the number of log records for which the processing +// has finished, either successful or failed. +type SDKProcessorLogProcessedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKProcessorLogProcessedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of log records for which the processing has finished, either successful or failed."), + metric.WithUnit("{log_record}"), +} + +// NewSDKProcessorLogProcessedObservable returns a new +// SDKProcessorLogProcessedObservable instrument. +func NewSDKProcessorLogProcessedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKProcessorLogProcessedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKProcessorLogProcessedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKProcessorLogProcessedObservableOpts + } else { + opt = append(opt, newSDKProcessorLogProcessedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.processor.log.processed", + opt..., + ) + if err != nil { + return SDKProcessorLogProcessedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKProcessorLogProcessedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKProcessorLogProcessedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKProcessorLogProcessedObservable) Name() string { + return "otel.sdk.processor.log.processed" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKProcessorLogProcessedObservable) Unit() string { + return "{log_record}" +} + +// Description returns the semantic convention description of the instrument +func (SDKProcessorLogProcessedObservable) Description() string { + return "The number of log records for which the processing has finished, either successful or failed." +} + +// AttrErrorType returns an optional attribute for the "error.type" semantic +// convention. It represents a low-cardinality description of the failure reason. +// SDK Batching Log Record Processors MUST use `queue_full` for log records +// dropped due to a full queue. +func (SDKProcessorLogProcessedObservable) AttrErrorType(val ErrorTypeAttr) attribute.KeyValue { + return attribute.String("error.type", string(val)) +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKProcessorLogProcessedObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKProcessorLogProcessedObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + // SDKProcessorLogQueueCapacity is an instrument used to record metric values // conforming to the "otel.sdk.processor.log.queue.capacity" semantic // conventions. It represents the maximum number of log records the queue of a @@ -1768,6 +2533,9 @@ func (m SDKProcessorSpanProcessed) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1775,6 +2543,7 @@ func (m SDKProcessorSpanProcessed) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1797,6 +2566,9 @@ func (m SDKProcessorSpanProcessed) Add( // processed already when it has been submitted to the exporter, not when the // corresponding export call has finished. func (m SDKProcessorSpanProcessed) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -1804,6 +2576,7 @@ func (m SDKProcessorSpanProcessed) AddSet(ctx context.Context, incr int64, set a o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -1834,6 +2607,88 @@ func (SDKProcessorSpanProcessed) AttrComponentType(val ComponentTypeAttr) attrib return attribute.String("otel.component.type", string(val)) } +// SDKProcessorSpanProcessedObservable is an instrument used to record metric +// values conforming to the "otel.sdk.processor.span.processed" semantic +// conventions. It represents the number of spans for which the processing has +// finished, either successful or failed. +type SDKProcessorSpanProcessedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKProcessorSpanProcessedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of spans for which the processing has finished, either successful or failed."), + metric.WithUnit("{span}"), +} + +// NewSDKProcessorSpanProcessedObservable returns a new +// SDKProcessorSpanProcessedObservable instrument. +func NewSDKProcessorSpanProcessedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKProcessorSpanProcessedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKProcessorSpanProcessedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKProcessorSpanProcessedObservableOpts + } else { + opt = append(opt, newSDKProcessorSpanProcessedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.processor.span.processed", + opt..., + ) + if err != nil { + return SDKProcessorSpanProcessedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKProcessorSpanProcessedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKProcessorSpanProcessedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKProcessorSpanProcessedObservable) Name() string { + return "otel.sdk.processor.span.processed" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKProcessorSpanProcessedObservable) Unit() string { + return "{span}" +} + +// Description returns the semantic convention description of the instrument +func (SDKProcessorSpanProcessedObservable) Description() string { + return "The number of spans for which the processing has finished, either successful or failed." +} + +// AttrErrorType returns an optional attribute for the "error.type" semantic +// convention. It represents a low-cardinality description of the failure reason. +// SDK Batching Span Processors MUST use `queue_full` for spans dropped due to a +// full queue. +func (SDKProcessorSpanProcessedObservable) AttrErrorType(val ErrorTypeAttr) attribute.KeyValue { + return attribute.String("error.type", string(val)) +} + +// AttrComponentName returns an optional attribute for the "otel.component.name" +// semantic convention. It represents a name uniquely identifying the instance of +// the OpenTelemetry component within its containing SDK instance. +func (SDKProcessorSpanProcessedObservable) AttrComponentName(val string) attribute.KeyValue { + return attribute.String("otel.component.name", val) +} + +// AttrComponentType returns an optional attribute for the "otel.component.type" +// semantic convention. It represents a name identifying the type of the +// OpenTelemetry component. +func (SDKProcessorSpanProcessedObservable) AttrComponentType(val ComponentTypeAttr) attribute.KeyValue { + return attribute.String("otel.component.type", string(val)) +} + // SDKProcessorSpanQueueCapacity is an instrument used to record metric values // conforming to the "otel.sdk.processor.span.queue.capacity" semantic // conventions. It represents the maximum number of spans the queue of a given @@ -2049,6 +2904,9 @@ func (m SDKSpanLive) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -2056,6 +2914,7 @@ func (m SDKSpanLive) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -2072,6 +2931,9 @@ func (m SDKSpanLive) Add( // AddSet adds incr to the existing count for set. func (m SDKSpanLive) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64UpDownCounter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64UpDownCounter.Add(ctx, incr) return @@ -2079,6 +2941,7 @@ func (m SDKSpanLive) AddSet(ctx context.Context, incr int64, set attribute.Set) o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -2094,6 +2957,72 @@ func (SDKSpanLive) AttrSpanSamplingResult(val SpanSamplingResultAttr) attribute. return attribute.String("otel.span.sampling_result", string(val)) } +// SDKSpanLiveObservable is an instrument used to record metric values conforming +// to the "otel.sdk.span.live" semantic conventions. It represents the number of +// created spans with `recording=true` for which the end operation has not been +// called yet. +type SDKSpanLiveObservable struct { + metric.Int64ObservableUpDownCounter +} + +var newSDKSpanLiveObservableOpts = []metric.Int64ObservableUpDownCounterOption{ + metric.WithDescription("The number of created spans with `recording=true` for which the end operation has not been called yet."), + metric.WithUnit("{span}"), +} + +// NewSDKSpanLiveObservable returns a new SDKSpanLiveObservable instrument. +func NewSDKSpanLiveObservable( + m metric.Meter, + opt ...metric.Int64ObservableUpDownCounterOption, +) (SDKSpanLiveObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKSpanLiveObservable{noop.Int64ObservableUpDownCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKSpanLiveObservableOpts + } else { + opt = append(opt, newSDKSpanLiveObservableOpts...) + } + + i, err := m.Int64ObservableUpDownCounter( + "otel.sdk.span.live", + opt..., + ) + if err != nil { + return SDKSpanLiveObservable{noop.Int64ObservableUpDownCounter{}}, err + } + return SDKSpanLiveObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKSpanLiveObservable) Inst() metric.Int64ObservableUpDownCounter { + return m.Int64ObservableUpDownCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKSpanLiveObservable) Name() string { + return "otel.sdk.span.live" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKSpanLiveObservable) Unit() string { + return "{span}" +} + +// Description returns the semantic convention description of the instrument +func (SDKSpanLiveObservable) Description() string { + return "The number of created spans with `recording=true` for which the end operation has not been called yet." +} + +// AttrSpanSamplingResult returns an optional attribute for the +// "otel.span.sampling_result" semantic convention. It represents the result +// value of the sampler for this span. +func (SDKSpanLiveObservable) AttrSpanSamplingResult(val SpanSamplingResultAttr) attribute.KeyValue { + return attribute.String("otel.span.sampling_result", string(val)) +} + // SDKSpanStarted is an instrument used to record metric values conforming to the // "otel.sdk.span.started" semantic conventions. It represents the number of // created spans. @@ -2163,6 +3092,9 @@ func (m SDKSpanStarted) Add( incr int64, attrs ...attribute.KeyValue, ) { + if !m.Int64Counter.Enabled(ctx) { + return + } if len(attrs) == 0 { m.Int64Counter.Add(ctx, incr) return @@ -2170,6 +3102,7 @@ func (m SDKSpanStarted) Add( o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -2189,6 +3122,9 @@ func (m SDKSpanStarted) Add( // Implementations MUST record this metric for all spans, even for non-recording // ones. func (m SDKSpanStarted) AddSet(ctx context.Context, incr int64, set attribute.Set) { + if !m.Int64Counter.Enabled(ctx) { + return + } if set.Len() == 0 { m.Int64Counter.Add(ctx, incr) return @@ -2196,6 +3132,7 @@ func (m SDKSpanStarted) AddSet(ctx context.Context, incr int64, set attribute.Se o := addOptPool.Get().(*[]metric.AddOption) defer func() { + clear(*o) *o = (*o)[:0] addOptPool.Put(o) }() @@ -2220,3 +3157,78 @@ func (SDKSpanStarted) AttrSpanParentOrigin(val SpanParentOriginAttr) attribute.K func (SDKSpanStarted) AttrSpanSamplingResult(val SpanSamplingResultAttr) attribute.KeyValue { return attribute.String("otel.span.sampling_result", string(val)) } + +// SDKSpanStartedObservable is an instrument used to record metric values +// conforming to the "otel.sdk.span.started" semantic conventions. It represents +// the number of created spans. +type SDKSpanStartedObservable struct { + metric.Int64ObservableCounter +} + +var newSDKSpanStartedObservableOpts = []metric.Int64ObservableCounterOption{ + metric.WithDescription("The number of created spans."), + metric.WithUnit("{span}"), +} + +// NewSDKSpanStartedObservable returns a new SDKSpanStartedObservable instrument. +func NewSDKSpanStartedObservable( + m metric.Meter, + opt ...metric.Int64ObservableCounterOption, +) (SDKSpanStartedObservable, error) { + // Check if the meter is nil. + if m == nil { + return SDKSpanStartedObservable{noop.Int64ObservableCounter{}}, nil + } + + if len(opt) == 0 { + opt = newSDKSpanStartedObservableOpts + } else { + opt = append(opt, newSDKSpanStartedObservableOpts...) + } + + i, err := m.Int64ObservableCounter( + "otel.sdk.span.started", + opt..., + ) + if err != nil { + return SDKSpanStartedObservable{noop.Int64ObservableCounter{}}, err + } + return SDKSpanStartedObservable{i}, nil +} + +// Inst returns the underlying metric instrument. +func (m SDKSpanStartedObservable) Inst() metric.Int64ObservableCounter { + return m.Int64ObservableCounter +} + +// Name returns the semantic convention name of the instrument. +func (SDKSpanStartedObservable) Name() string { + return "otel.sdk.span.started" +} + +// Unit returns the semantic convention unit of the instrument +func (SDKSpanStartedObservable) Unit() string { + return "{span}" +} + +// Description returns the semantic convention description of the instrument +func (SDKSpanStartedObservable) Description() string { + return "The number of created spans." +} + +// AttrSpanParentOrigin returns an optional attribute for the +// "otel.span.parent.origin" semantic convention. It represents the determines +// whether the span has a parent span, and if so, [whether it is a remote parent] +// . +// +// [whether it is a remote parent]: https://opentelemetry.io/docs/specs/otel/trace/api/#isremote +func (SDKSpanStartedObservable) AttrSpanParentOrigin(val SpanParentOriginAttr) attribute.KeyValue { + return attribute.String("otel.span.parent.origin", string(val)) +} + +// AttrSpanSamplingResult returns an optional attribute for the +// "otel.span.sampling_result" semantic convention. It represents the result +// value of the sampler for this span. +func (SDKSpanStartedObservable) AttrSpanSamplingResult(val SpanSamplingResultAttr) attribute.KeyValue { + return attribute.String("otel.span.sampling_result", string(val)) +} diff --git a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/schema.go b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/schema.go similarity index 73% rename from vendor/go.opentelemetry.io/otel/semconv/v1.40.0/schema.go rename to vendor/go.opentelemetry.io/otel/semconv/v1.41.0/schema.go index a07ffa336..24948a48f 100644 --- a/vendor/go.opentelemetry.io/otel/semconv/v1.40.0/schema.go +++ b/vendor/go.opentelemetry.io/otel/semconv/v1.41.0/schema.go @@ -1,9 +1,11 @@ +// Code generated from semantic convention specification. DO NOT EDIT. + // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.40.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.41.0" // SchemaURL is the schema URL that matches the version of the semantic conventions // that this package defines. Semconv packages starting from v1.4.0 must declare // non-empty schema URL in the form https://opentelemetry.io/schemas/ -const SchemaURL = "https://opentelemetry.io/schemas/1.40.0" +const SchemaURL = "https://opentelemetry.io/schemas/1.41.0" diff --git a/vendor/go.opentelemetry.io/otel/trace/auto.go b/vendor/go.opentelemetry.io/otel/trace/auto.go index 9316fd0ac..a75cf047d 100644 --- a/vendor/go.opentelemetry.io/otel/trace/auto.go +++ b/vendor/go.opentelemetry.io/otel/trace/auto.go @@ -20,7 +20,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" "go.opentelemetry.io/otel/trace/embedded" "go.opentelemetry.io/otel/trace/internal/telemetry" ) @@ -314,6 +314,14 @@ func convAttrValue(value attribute.Value) telemetry.Value { case attribute.STRING: v := truncate(maxSpan.AttrValueLen, value.AsString()) return telemetry.StringValue(v) + case attribute.BYTESLICE: + // len(v.AsString()) is identical to len(v.AsByteSlice()) but + // avoids allocating the full slice before truncation. + s := value.AsString() + if maxSpan.AttrValueLen >= 0 && len(s) > maxSpan.AttrValueLen { + return telemetry.BytesValue([]byte(s[:maxSpan.AttrValueLen])) + } + return telemetry.BytesValue([]byte(s)) case attribute.BOOLSLICE: slice := value.AsBoolSlice() out := make([]telemetry.Value, 0, len(slice)) @@ -343,6 +351,13 @@ func convAttrValue(value attribute.Value) telemetry.Value { out = append(out, telemetry.StringValue(v)) } return telemetry.SliceValue(out...) + case attribute.SLICE: + slice := value.AsSlice() + out := make([]telemetry.Value, 0, len(slice)) + for _, v := range slice { + out = append(out, convAttrValue(v)) + } + return telemetry.SliceValue(out...) } return telemetry.Value{} } @@ -463,7 +478,8 @@ func (s *autoSpan) RecordError(err error, opts ...EventOption) { cfg := NewEventConfig(opts...) attrs := cfg.Attributes() - attrs = append(attrs, + attrs = append( + attrs, semconv.ExceptionType(typeStr(err)), semconv.ExceptionMessage(err.Error()), ) diff --git a/vendor/go.opentelemetry.io/otel/trace/config.go b/vendor/go.opentelemetry.io/otel/trace/config.go index d9ecef1ca..4cedba5ac 100644 --- a/vendor/go.opentelemetry.io/otel/trace/config.go +++ b/vendor/go.opentelemetry.io/otel/trace/config.go @@ -34,10 +34,17 @@ func (t *TracerConfig) SchemaURL() string { return t.schemaURL } +type experimentalOption interface { + Experimental() +} + // NewTracerConfig applies all the options to a returned TracerConfig. func NewTracerConfig(options ...TracerOption) TracerConfig { var config TracerConfig for _, option := range options { + if _, ok := option.(experimentalOption); ok { + continue + } config = option.apply(config) } return config @@ -103,6 +110,9 @@ func (cfg *SpanConfig) SpanKind() SpanKind { func NewSpanStartConfig(options ...SpanStartOption) SpanConfig { var c SpanConfig for _, option := range options { + if _, ok := option.(experimentalOption); ok { + continue + } c = option.applySpanStart(c) } return c @@ -115,6 +125,9 @@ func NewSpanStartConfig(options ...SpanStartOption) SpanConfig { func NewSpanEndConfig(options ...SpanEndOption) SpanConfig { var c SpanConfig for _, option := range options { + if _, ok := option.(experimentalOption); ok { + continue + } c = option.applySpanEnd(c) } return c @@ -167,6 +180,9 @@ func (cfg *EventConfig) StackTrace() bool { func NewEventConfig(options ...EventOption) EventConfig { var c EventConfig for _, option := range options { + if _, ok := option.(experimentalOption); ok { + continue + } c = option.applyEvent(c) } if c.timestamp.IsZero() { diff --git a/vendor/go.opentelemetry.io/otel/trace/internal/telemetry/span.go b/vendor/go.opentelemetry.io/otel/trace/internal/telemetry/span.go index e7ca62c66..61c7819a2 100644 --- a/vendor/go.opentelemetry.io/otel/trace/internal/telemetry/span.go +++ b/vendor/go.opentelemetry.io/otel/trace/internal/telemetry/span.go @@ -314,9 +314,9 @@ type SpanEvent struct { } // MarshalJSON encodes e into OTLP formatted JSON. -func (e SpanEvent) MarshalJSON() ([]byte, error) { - t := e.Time.UnixNano() - if e.Time.IsZero() || t < 0 { +func (se SpanEvent) MarshalJSON() ([]byte, error) { + t := se.Time.UnixNano() + if se.Time.IsZero() || t < 0 { t = 0 } @@ -325,7 +325,7 @@ func (e SpanEvent) MarshalJSON() ([]byte, error) { Alias Time uint64 `json:"timeUnixNano,omitempty"` }{ - Alias: Alias(e), + Alias: Alias(se), Time: uint64(t), // nolint: gosec // >0 checked above }) } diff --git a/vendor/go.opentelemetry.io/otel/version.go b/vendor/go.opentelemetry.io/otel/version.go index 1db4f47e4..72746acfd 100644 --- a/vendor/go.opentelemetry.io/otel/version.go +++ b/vendor/go.opentelemetry.io/otel/version.go @@ -5,5 +5,5 @@ package otel // import "go.opentelemetry.io/otel" // Version is the current release version of OpenTelemetry in use. func Version() string { - return "1.43.0" + return "1.44.0" } diff --git a/vendor/go.opentelemetry.io/otel/versions.yaml b/vendor/go.opentelemetry.io/otel/versions.yaml index bcc6ee78a..d6dbf803e 100644 --- a/vendor/go.opentelemetry.io/otel/versions.yaml +++ b/vendor/go.opentelemetry.io/otel/versions.yaml @@ -3,7 +3,7 @@ module-sets: stable-v1: - version: v1.43.0 + version: v1.44.0 modules: - go.opentelemetry.io/otel - go.opentelemetry.io/otel/bridge/opencensus @@ -22,11 +22,12 @@ module-sets: - go.opentelemetry.io/otel/sdk/metric - go.opentelemetry.io/otel/trace experimental-metrics: - version: v0.65.0 + version: v0.66.0 modules: - go.opentelemetry.io/otel/exporters/prometheus + - go.opentelemetry.io/otel/metric/x experimental-logs: - version: v0.19.0 + version: v0.20.0 modules: - go.opentelemetry.io/otel/log - go.opentelemetry.io/otel/log/logtest @@ -36,7 +37,7 @@ module-sets: - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp - go.opentelemetry.io/otel/exporters/stdout/stdoutlog experimental-schema: - version: v0.0.16 + version: v0.0.17 modules: - go.opentelemetry.io/otel/schema excluded-modules: @@ -55,6 +56,9 @@ modules: go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc: version-refs: - ./internal/version.go + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp: + version-refs: + - ./internal/version.go go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc: version-refs: - ./internal/version.go diff --git a/vendor/go.opentelemetry.io/proto/otlp/common/v1/common.pb.go b/vendor/go.opentelemetry.io/proto/otlp/common/v1/common.pb.go index 304f64763..cef358000 100644 --- a/vendor/go.opentelemetry.io/proto/otlp/common/v1/common.pb.go +++ b/vendor/go.opentelemetry.io/proto/otlp/common/v1/common.pb.go @@ -194,7 +194,7 @@ type AnyValue_StringValueStrindex struct { // Profiling signal and process the data as if this value were absent or // empty, ignoring its semantic content for the non-Profiling signal. // - // Status: [Development] + // Status: [Alpha] StringValueStrindex int32 `protobuf:"varint,8,opt,name=string_value_strindex,json=stringValueStrindex,proto3,oneof"` } @@ -330,7 +330,7 @@ type KeyValue struct { unknownFields protoimpl.UnknownFields // The key name of the pair. - // key_ref MUST NOT be set if key is used. + // key_strindex MUST NOT be set if key is used. Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // The value of the pair. Value *AnyValue `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` @@ -344,7 +344,7 @@ type KeyValue struct { // Profiling signal and process the data as if this value were absent or // empty, ignoring its semantic content for the non-Profiling signal. // - // Status: [Development] + // Status: [Alpha] KeyStrindex int32 `protobuf:"varint,3,opt,name=key_strindex,json=keyStrindex,proto3" json:"key_strindex,omitempty"` } diff --git a/vendor/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go b/vendor/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go index 748c9fb1c..e148ddd13 100644 --- a/vendor/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go +++ b/vendor/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go @@ -527,7 +527,7 @@ type Metric struct { // A description of the metric, which can be used in documentation. Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` // The unit in which the metric value is reported. Follows the format - // described by https://unitsofmeasure.org/ucum.html. + // described by https://ucum.org/ucum and https://units-of-measurement.org/ Unit string `protobuf:"bytes,3,opt,name=unit,proto3" json:"unit,omitempty"` // Data determines the aggregation type (if any) of the metric, what is the // reported value type for the data points, as well as the relatationship to diff --git a/vendor/go.yaml.in/yaml/v3/parserc.go b/vendor/go.yaml.in/yaml/v3/parserc.go index 25fe82363..f35829db4 100644 --- a/vendor/go.yaml.in/yaml/v3/parserc.go +++ b/vendor/go.yaml.in/yaml/v3/parserc.go @@ -226,9 +226,9 @@ func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool } // Parse the production: -// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END // -// ************ +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -249,13 +249,11 @@ func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) } // Parse the productions: -// implicit_document ::= block_node DOCUMENT-END* -// -// * // -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* -// -// ************************* +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { token := peek_token(parser) @@ -359,9 +357,9 @@ func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t } // Parse the productions: -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* // -// *********** +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -382,11 +380,10 @@ func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event } // Parse the productions: -// implicit_document ::= block_node DOCUMENT-END* -// -// ************* // -// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -432,42 +429,32 @@ func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) } // Parse the productions: -// block_node_or_indentless_sequence ::= -// -// ALIAS -// ***** -// | properties (block_content | indentless_block_sequence)? -// ********** * -// | block_content | indentless_block_sequence -// * -// -// block_node ::= ALIAS -// -// ***** -// | properties block_content? -// ********** * -// | block_content -// * -// -// flow_node ::= ALIAS -// -// ***** -// | properties flow_content? -// ********** * -// | flow_content -// * -// -// properties ::= TAG ANCHOR? | ANCHOR TAG? -// -// ************************* -// -// block_content ::= block_collection | flow_collection | SCALAR -// -// ****** // -// flow_content ::= flow_collection | SCALAR -// -// ****** +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() @@ -697,9 +684,9 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i } // Parse the productions: -// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END // -// ******************** *********** * ********* +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -755,9 +742,9 @@ func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_e } // Parse the productions: -// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ // -// *********** * +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -821,15 +808,15 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { } // Parse the productions: -// block_mapping ::= BLOCK-MAPPING_START // -// ******************* -// ((KEY block_node_or_indentless_sequence?)? -// *** * -// (VALUE block_node_or_indentless_sequence?)?)* +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* // -// BLOCK-END -// ********* +// BLOCK-END +// ********* func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -896,13 +883,14 @@ func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_even } // Parse the productions: -// block_mapping ::= BLOCK-MAPPING_START // -// ((KEY block_node_or_indentless_sequence?)? +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? // -// (VALUE block_node_or_indentless_sequence?)?)* -// ***** * -// BLOCK-END +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -929,19 +917,17 @@ func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_ev } // Parse the productions: -// flow_sequence ::= FLOW-SEQUENCE-START -// -// ******************* -// (flow_sequence_entry FLOW-ENTRY)* -// * ********** -// flow_sequence_entry? -// * -// FLOW-SEQUENCE-END -// ***************** // -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// -// * +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -1005,9 +991,9 @@ func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_ev } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// *** * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1026,9 +1012,9 @@ func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, ev } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// ***** * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1050,9 +1036,9 @@ func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, } // Parse the productions: -// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? // -// * +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { token := peek_token(parser) if token == nil { @@ -1068,18 +1054,17 @@ func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, ev } // Parse the productions: -// flow_mapping ::= FLOW-MAPPING-START -// -// ****************** -// (flow_mapping_entry FLOW-ENTRY)* -// * ********** -// flow_mapping_entry? -// ****************** -// FLOW-MAPPING-END -// **************** // -// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// - *** * +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) @@ -1144,8 +1129,9 @@ func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event } // Parse the productions: -// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? -// - ***** * +// +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { token := peek_token(parser) if token == nil { diff --git a/vendor/go.yaml.in/yaml/v3/yamlh.go b/vendor/go.yaml.in/yaml/v3/yamlh.go index f59aa40f6..07c442361 100644 --- a/vendor/go.yaml.in/yaml/v3/yamlh.go +++ b/vendor/go.yaml.in/yaml/v3/yamlh.go @@ -433,21 +433,19 @@ type yaml_document_t struct { // The prototype of a read handler. // -// The read handler is called when the parser needs to read more bytes from the -// source. The handler should write not more than size bytes to the buffer. -// The number of written bytes should be set to the size_read variable. +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. // -// [in,out] data A pointer to an application data specified by +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. // -// yaml_parser_set_input(). -// -// [out] buffer The buffer to write the data from the source. -// [in] size The size of the buffer. -// [out] size_read The actual number of bytes read from the source. -// -// On success, the handler should return 1. If the handler failed, -// the returned value should be 0. On EOF, the handler should set the -// size_read to 0 and return 1. +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) // This structure holds information about a potential simple key. @@ -655,19 +653,17 @@ type yaml_comment_t struct { // The prototype of a write handler. // -// The write handler is called when the emitter needs to flush the accumulated -// characters to the output. The handler should write @a size bytes of the -// @a buffer to the output. -// -// @param[in,out] data A pointer to an application data specified by -// -// yaml_emitter_set_output(). +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. // -// @param[in] buffer The buffer with bytes to be written. -// @param[in] size The size of the buffer. +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. // -// @returns On success, the handler should return @c 1. If the handler failed, -// the returned value should be @c 0. +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error type yaml_emitter_state_t int diff --git a/vendor/golang.org/x/net/html/entity.go b/vendor/golang.org/x/net/html/entity.go index b628880a0..4e8d5d55f 100644 --- a/vendor/golang.org/x/net/html/entity.go +++ b/vendor/golang.org/x/net/html/entity.go @@ -2156,9 +2156,8 @@ var entity = map[string]rune{ // HTML entities that are two unicode codepoints. var entity2 = map[string][2]rune{ - // TODO(nigeltao): Handle replacements that are wider than their names. - // "nLt;": {'\u226A', '\u20D2'}, - // "nGt;": {'\u226B', '\u20D2'}, + "nLt;": {'\u226A', '\u20D2'}, + "nGt;": {'\u226B', '\u20D2'}, "NotEqualTilde;": {'\u2242', '\u0338'}, "NotGreaterFullEqual;": {'\u2267', '\u0338'}, "NotGreaterGreater;": {'\u226B', '\u0338'}, diff --git a/vendor/golang.org/x/net/html/escape.go b/vendor/golang.org/x/net/html/escape.go index 12f227370..df3edc5b1 100644 --- a/vendor/golang.org/x/net/html/escape.go +++ b/vendor/golang.org/x/net/html/escape.go @@ -6,6 +6,7 @@ package html import ( "bytes" + "slices" "strings" "unicode/utf8" ) @@ -50,25 +51,24 @@ var replacementTable = [...]rune{ // 0x0D->'\u000D' is a no-op. } -// unescapeEntity reads an entity like "<" from b[src:] and writes the -// corresponding "<" to b[dst:], returning the incremented dst and src cursors. -// Precondition: b[src] == '&' && dst <= src. -// attribute should be true if parsing an attribute value. -func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { +// unescapeEntity attempts to consume a character reference from s[src:], +// returning the rune, potential second rune, and number of bytes consumed +// (which indicates the length of the character reference). It is assumed that +// the first byte of s is '&'. attribute should be true if parsing an attribute +// value. +func unescapeEntity(s []byte, attribute bool) (rune, rune, int) { // https://html.spec.whatwg.org/multipage/syntax.html#consume-a-character-reference // i starts at 1 because we already know that s[0] == '&'. - i, s := 1, b[src:] + i := 1 if len(s) <= 1 { - b[dst] = b[src] - return dst + 1, src + 1 + return '&', 0, 1 } if s[i] == '#' { - if len(s) <= 3 { // We need to have at least "&#.". - b[dst] = b[src] - return dst + 1, src + 1 + if len(s) <= 2 { // We need to have at least "&#". + return '&', 0, 1 } i++ c := s[i] @@ -78,34 +78,43 @@ func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { i++ } + i0 := i x := '\x00' for i < len(s) { c = s[i] - i++ + var d rune + var mult rune if hex { + mult = 16 if '0' <= c && c <= '9' { - x = 16*x + rune(c) - '0' - continue + d = rune(c) - '0' } else if 'a' <= c && c <= 'f' { - x = 16*x + rune(c) - 'a' + 10 - continue + d = rune(c) - 'a' + 10 } else if 'A' <= c && c <= 'F' { - x = 16*x + rune(c) - 'A' + 10 - continue + d = rune(c) - 'A' + 10 + } else { + break + } + } else { + mult = 10 + if '0' <= c && c <= '9' { + d = rune(c) - '0' + } else { + break } - } else if '0' <= c && c <= '9' { - x = 10*x + rune(c) - '0' - continue } - if c != ';' { - i-- + if x <= 0x10FFFF { + x = mult*x + d } - break + i++ + } + + if i == i0 { // No characters matched. + return '&', 0, 1 } - if i <= 3 { // No characters matched. - b[dst] = b[src] - return dst + 1, src + 1 + if i < len(s) && s[i] == ';' { + i++ } if 0x80 <= x && x <= 0x9F { @@ -116,7 +125,7 @@ func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { x = '\uFFFD' } - return dst + utf8.EncodeRune(b[dst:], x), src + i + return x, 0, i } // Consume the maximum number of characters possible, with the @@ -141,10 +150,9 @@ func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { } else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' { // No-op. } else if x := entity[entityName]; x != 0 { - return dst + utf8.EncodeRune(b[dst:], x), src + i + return x, 0, i } else if x := entity2[entityName]; x[0] != 0 { - dst1 := dst + utf8.EncodeRune(b[dst:], x[0]) - return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i + return x[0], x[1], i } else if !attribute { maxLen := len(entityName) - 1 if maxLen > longestEntityWithoutSemicolon { @@ -152,35 +160,67 @@ func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) { } for j := maxLen; j > 1; j-- { if x := entity[entityName[:j]]; x != 0 { - return dst + utf8.EncodeRune(b[dst:], x), src + j + 1 + return x, 0, j + 1 } } } - dst1, src1 = dst+i, src+i - copy(b[dst:dst1], b[src:src1]) - return dst1, src1 + return '&', 0, 1 } -// unescape unescapes b's entities in-place, so that "a<b" becomes "a entityNameLen { + if reusingB { + out = slices.Clone(out) + reusingB = false } - return b[0:dst] + out = slices.Grow(out, replLen) + } + out = utf8.AppendRune(out, r1) + if r2 != 0 { + out = utf8.AppendRune(out, r2) } + + src += entityNameLen } - return b + + return out } // lower lower-cases the A-Z bytes in b in-place, so that "aBc" becomes "abc". diff --git a/vendor/golang.org/x/net/html/foreign.go b/vendor/golang.org/x/net/html/foreign.go index e8515d8e8..65d01d1ed 100644 --- a/vendor/golang.org/x/net/html/foreign.go +++ b/vendor/golang.org/x/net/html/foreign.go @@ -23,7 +23,7 @@ func adjustForeignAttributes(aa []Attribute) { } switch a.Key { case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", - "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink": + "xlink:title", "xlink:type", "xml:lang", "xml:space", "xmlns:xlink": j := strings.Index(a.Key, ":") aa[i].Namespace = a.Key[:j] aa[i].Key = a.Key[j+1:] diff --git a/vendor/golang.org/x/net/html/parse.go b/vendor/golang.org/x/net/html/parse.go index b3d2a2558..165b6108d 100644 --- a/vendor/golang.org/x/net/html/parse.go +++ b/vendor/golang.org/x/net/html/parse.go @@ -63,7 +63,7 @@ func (p *parser) top() *Node { // Stop tags for use in popUntil. These come from section 12.2.4.2. var ( defaultScopeStopTags = map[string][]a.Atom{ - "": {a.Applet, a.Caption, a.Html, a.Table, a.Td, a.Th, a.Marquee, a.Object, a.Template}, + "": {a.Applet, a.Caption, a.Html, a.Table, a.Td, a.Th, a.Marquee, a.Object, a.Template, a.Select}, "math": {a.AnnotationXml, a.Mi, a.Mn, a.Mo, a.Ms, a.Mtext}, "svg": {a.Desc, a.ForeignObject, a.Title}, } @@ -78,7 +78,6 @@ const ( tableScope tableRowScope tableBodyScope - selectScope ) // popUntil pops the stack of open elements at the highest element whose tag @@ -133,10 +132,6 @@ func (p *parser) indexOfElementInScope(s scope, matchTags ...a.Atom) int { if tagAtom == a.Html || tagAtom == a.Table || tagAtom == a.Template { return -1 } - case selectScope: - if tagAtom != a.Optgroup && tagAtom != a.Option { - return -1 - } default: panic(fmt.Sprintf("html: internal error: indexOfElementInScope unknown scope: %d", s)) } @@ -460,21 +455,6 @@ func (p *parser) resetInsertionMode() { } switch n.DataAtom { - case a.Select: - if !last { - for ancestor, first := n, p.oe[0]; ancestor != first; { - ancestor = p.oe[p.oe.index(ancestor)-1] - switch ancestor.DataAtom { - case a.Template: - p.im = inSelectIM - return - case a.Table: - p.im = inSelectInTableIM - return - } - } - } - p.im = inSelectIM case a.Td, a.Th: // TODO: remove this divergence from the HTML5 spec. // @@ -1002,7 +982,10 @@ func inBodyIM(p *parser) bool { p.popUntil(buttonScope, a.P) p.addElement() case a.Button: - p.popUntil(defaultScope, a.Button) + if p.elementInScope(defaultScope, a.Button) { + p.generateImpliedEndTags() + p.popUntil(defaultScope, a.Button) + } p.reconstructActiveFormattingElements() p.addElement() p.framesetOK = false @@ -1040,7 +1023,18 @@ func inBodyIM(p *parser) bool { p.framesetOK = false p.im = inTableIM return true - case a.Area, a.Br, a.Embed, a.Img, a.Input, a.Keygen, a.Wbr: + case a.Area, a.Br, a.Embed, a.Img, a.Keygen, a.Wbr: + p.reconstructActiveFormattingElements() + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + p.framesetOK = false + case a.Input: + if p.fragment && p.context.DataAtom == a.Select { + // Ignore the token. + return true + } + p.popUntil(defaultScope, a.Select) p.reconstructActiveFormattingElements() p.addElement() p.oe.pop() @@ -1061,7 +1055,13 @@ func inBodyIM(p *parser) bool { p.oe.pop() p.acknowledgeSelfClosingTag() case a.Hr: - p.popUntil(buttonScope, a.P) + if p.elementInScope(buttonScope, a.P) { + p.generateImpliedEndTags("p") + p.popUntil(defaultScope, a.P) + } + if p.elementInScope(defaultScope, a.Select) { + p.generateImpliedEndTags() + } p.addElement() p.oe.pop() p.acknowledgeSelfClosingTag() @@ -1095,13 +1095,30 @@ func inBodyIM(p *parser) bool { // Don't let the tokenizer go into raw text mode when scripting is disabled. p.tokenizer.NextIsNotRawText() case a.Select: + if p.fragment && p.context.DataAtom == a.Select { + // Ignore the token. + return true + } else if p.popUntil(defaultScope, a.Select) { + return true + } p.reconstructActiveFormattingElements() p.addElement() p.framesetOK = false - p.im = inSelectIM return true - case a.Optgroup, a.Option: - if p.top().DataAtom == a.Option { + case a.Option: + if p.elementInScope(defaultScope, a.Select) { + p.generateImpliedEndTags("optgroup") + // If oe has option element in scope, parse error? + } else if p.top().DataAtom == a.Option { + p.oe.pop() + } + p.reconstructActiveFormattingElements() + p.addElement() + case a.Optgroup: + if p.elementInScope(defaultScope, a.Select) { + p.generateImpliedEndTags() + // If oe has option or optgroup element in scope, parse error? + } else if p.top().DataAtom == a.Option { p.oe.pop() } p.reconstructActiveFormattingElements() @@ -1149,7 +1166,12 @@ func inBodyIM(p *parser) bool { return false } return true - case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dialog, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Main, a.Menu, a.Nav, a.Ol, a.Pre, a.Search, a.Section, a.Summary, a.Ul: + case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dialog, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Main, a.Menu, a.Nav, a.Ol, a.Pre, a.Search, a.Section, a.Select, a.Summary, a.Ul: + if !p.elementInScope(defaultScope, p.tok.DataAtom) { + // Ignore the token. + return true + } + p.generateImpliedEndTags() p.popUntil(defaultScope, p.tok.DataAtom) case a.Form: if p.oe.contains(a.Template) { @@ -1488,17 +1510,6 @@ func inTableIM(p *parser) bool { } p.addElement() p.form = p.oe.pop() - case a.Select: - p.reconstructActiveFormattingElements() - switch p.top().DataAtom { - case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: - p.fosterParenting = true - } - p.addElement() - p.fosterParenting = false - p.framesetOK = false - p.im = inSelectInTableIM - return true } case EndTagToken: switch p.tok.DataAtom { @@ -1547,12 +1558,6 @@ func inCaptionIM(p *parser) bool { p.clearActiveFormattingElements() p.im = inTableIM return false - case a.Select: - p.reconstructActiveFormattingElements() - p.addElement() - p.framesetOK = false - p.im = inSelectInTableIM - return true } case EndTagToken: switch p.tok.DataAtom { @@ -1762,12 +1767,6 @@ func inCellIM(p *parser) bool { } // Ignore the token. return true - case a.Select: - p.reconstructActiveFormattingElements() - p.addElement() - p.framesetOK = false - p.im = inSelectInTableIM - return true } case EndTagToken: switch p.tok.DataAtom { @@ -1798,118 +1797,6 @@ func inCellIM(p *parser) bool { return inBodyIM(p) } -// Section 12.2.6.4.16. -func inSelectIM(p *parser) bool { - switch p.tok.Type { - case TextToken: - p.addText(strings.Replace(p.tok.Data, "\x00", "", -1)) - case StartTagToken: - switch p.tok.DataAtom { - case a.Html: - return inBodyIM(p) - case a.Option: - if p.top().DataAtom == a.Option { - p.oe.pop() - } - p.addElement() - case a.Optgroup: - if p.top().DataAtom == a.Option { - p.oe.pop() - } - if p.top().DataAtom == a.Optgroup { - p.oe.pop() - } - p.addElement() - case a.Select: - if !p.popUntil(selectScope, a.Select) { - // Ignore the token. - return true - } - p.resetInsertionMode() - case a.Input, a.Keygen, a.Textarea: - if p.elementInScope(selectScope, a.Select) { - p.parseImpliedToken(EndTagToken, a.Select, a.Select.String()) - return false - } - // In order to properly ignore