From ec8160fb933b66fba815ec674b7d5e40ed404e74 Mon Sep 17 00:00:00 2001 From: Chirag Gajjar Date: Thu, 6 Aug 2026 11:03:39 +0000 Subject: [PATCH] Revert "taskqueue: Implement Cloud Tasks v2beta3 backend for App Engine push queues with transactional outbox (#407)" This reverts commit af6b28078e3fe97b3e3964556a03a1ad73898416. --- go.mod | 42 +- go.sum | 132 ++---- internal/transaction.go | 27 -- taskqueue/cloudtask.go | 586 --------------------------- taskqueue/cloudtask_transactional.go | 242 ----------- taskqueue/taskqueue.go | 13 - 6 files changed, 40 insertions(+), 1002 deletions(-) delete mode 100644 taskqueue/cloudtask.go delete mode 100644 taskqueue/cloudtask_transactional.go diff --git a/go.mod b/go.mod index 5cc9fcad..c6744ed5 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,9 @@ module google.golang.org/appengine -go 1.25.0 +go 1.11 require ( - cloud.google.com/go/cloudtasks v1.19.0 - github.com/golang/protobuf v1.5.4 - golang.org/x/text v0.38.0 - google.golang.org/protobuf v1.36.11 -) - -require ( - cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.20.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.11.0 // indirect - cloud.google.com/go/longrunning v1.2.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect - github.com/googleapis/gax-go/v2 v2.23.0 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/net v0.56.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/time v0.15.0 // indirect - google.golang.org/api v0.287.1 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/grpc v1.82.0 // indirect + github.com/golang/protobuf v1.5.2 + golang.org/x/text v0.3.8 + google.golang.org/protobuf v1.33.0 ) diff --git a/go.sum b/go.sum index f9ec04ce..31a039dc 100644 --- a/go.sum +++ b/go.sum @@ -1,96 +1,36 @@ -cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= -cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/cloudtasks v1.19.0 h1:+RK0lPIB6TlcBP7JyqmmhCNihp1Iw4QQ8uxcvlKhBVQ= -cloud.google.com/go/cloudtasks v1.19.0/go.mod h1:8q8wNubq0jFvXW5Pz8P3O7QWJBXOmfrY918FqTgIqHA= -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= -cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= -cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= -github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= -github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= -github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= -github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= -github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= -google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/internal/transaction.go b/internal/transaction.go index 1f1097b4..2ae8ab9f 100644 --- a/internal/transaction.go +++ b/internal/transaction.go @@ -17,23 +17,6 @@ import ( pb "google.golang.org/appengine/internal/datastore" ) -// Transaction hooks for taskqueue outbox pattern -var ( - PostCommitHook func(ctx context.Context, handle uint64) - RollbackHook func(handle uint64) -) - -func TransactionFromContext(ctx context.Context) *pb.Transaction { - if t := transactionFromContext(ctx); t != nil { - return &t.transaction - } - return nil -} - -func TransactionlessContext(ctx context.Context) context.Context { - return withTransaction(ctx, nil) -} - var transactionSetters = make(map[reflect.Type]reflect.Value) // RegisterTransactionSetter registers a function that sets transaction information @@ -106,9 +89,6 @@ func RunTransactionOnce(c context.Context, f func(context.Context) error, xg boo // Ignore the error return value, since we are already returning a non-nil // error (or we're panicking). Call(c, "datastore_v3", "Rollback", &t.transaction, &basepb.VoidProto{}) - if RollbackHook != nil { - RollbackHook(t.transaction.GetHandle()) - } }() if err := f(withTransaction(c, t)); err != nil { return &t.transaction, err @@ -131,12 +111,5 @@ func RunTransactionOnce(c context.Context, f func(context.Context) error, xg boo return &t.transaction, ErrConcurrentTransaction } } - if err == nil { - if PostCommitHook != nil { - PostCommitHook(c, t.transaction.GetHandle()) - } - } else if RollbackHook != nil { - RollbackHook(t.transaction.GetHandle()) - } return &t.transaction, err } diff --git a/taskqueue/cloudtask.go b/taskqueue/cloudtask.go deleted file mode 100644 index 10e67d78..00000000 --- a/taskqueue/cloudtask.go +++ /dev/null @@ -1,586 +0,0 @@ -package taskqueue - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "regexp" - "strconv" - "strings" - "time" - - "google.golang.org/appengine" - "google.golang.org/appengine/datastore" - "google.golang.org/appengine/internal" - pb "google.golang.org/appengine/internal/taskqueue" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" - - cloudtasks "cloud.google.com/go/cloudtasks/apiv2beta3" - taskspb "cloud.google.com/go/cloudtasks/apiv2beta3/cloudtaskspb" -) - -const ( - maxTaskPayloadBytes = 100 * 1024 // 100 KB max payload size for Cloud Tasks - maxTransactionalTasks = 5 // Maximum tasks allowed in a single Datastore transaction - batchCreateChunkSize = 100 // Maximum tasks per BatchCreateTasks request - batchDeleteChunkSize = 1000 // Maximum tasks per BatchDeleteTasks request - - grpcNotFound = 5 - grpcAlreadyExists = 6 - httpNotFound = 404 - httpAlreadyExists = 409 -) - -var ( - taskNameRegex = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) - ErrTooManyTasksInTransaction = &internal.APIError{ - Service: "taskqueue", - Detail: "too many tasks in transaction", - Code: int32(pb.TaskQueueServiceError_TOO_MANY_TASKS), - } -) - -func useCloudTasks() bool { - v, _ := strconv.ParseBool(os.Getenv("APPENGINE_USE_CLOUDTASK_PUSH_QUEUE")) - return v -} - -func newUnknownTaskError(detail string) error { - return &internal.APIError{ - Service: "taskqueue", - Detail: detail, - Code: int32(pb.TaskQueueServiceError_UNKNOWN_TASK), - } -} - -func isAlreadyExistsError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - return strings.Contains(msg, "AlreadyExists") || strings.Contains(msg, "already exists") || strings.Contains(msg, "409") || strings.Contains(msg, "Policy checks are unavailable") -} - -func isUnimplementedError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - return strings.Contains(msg, "Unimplemented") || strings.Contains(msg, "unknown method") || strings.Contains(msg, "404") -} - -func getQueuePath(ctx context.Context, queueName string) (string, error) { - if queueName == "" { - queueName = "default" - } - project := appengine.AppID(ctx) - if idx := strings.Index(project, "~"); idx != -1 { - project = project[idx+1:] - } - region, err := getRegion(ctx) - if err != nil { - return "", fmt.Errorf("failed to get region: %v", err) - } - return fmt.Sprintf("projects/%s/locations/%s/queues/%s", project, region, queueName), nil -} - -func getRegion(ctx context.Context) (string, error) { - req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/instance/region", nil) - if err != nil { - return "", err - } - req.Header.Set("Metadata-Flavor", "Google") - resp, err := http.DefaultClient.Do(req.WithContext(ctx)) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("metadata server returned status %d", resp.StatusCode) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - parts := strings.Split(strings.TrimSpace(string(body)), "/") - if len(parts) == 0 { - return "", fmt.Errorf("invalid region format: %s", string(body)) - } - return parts[len(parts)-1], nil -} - -func sendTask(ctx context.Context, queueName string, taskName string, taskObj *taskspb.Task) (string, error) { - parent, err := getQueuePath(ctx, queueName) - if err != nil { - return "", err - } - - client, err := cloudtasks.NewClient(ctx) - if err != nil { - return "", fmt.Errorf("failed to create cloudtasks client: %v", err) - } - defer client.Close() - - req := &taskspb.CreateTaskRequest{ - Parent: parent, - Task: taskObj, - } - - createdTask, err := client.CreateTask(ctx, req) - if err != nil { - if isAlreadyExistsError(err) { - return "", ErrTaskAlreadyAdded - } - return "", err - } - shortName := taskName - if createdTask != nil && createdTask.Name != "" { - if idx := strings.LastIndex(createdTask.Name, "/"); idx != -1 { - shortName = createdTask.Name[idx+1:] - } else { - shortName = createdTask.Name - } - } - return shortName, nil -} - -func extractServiceFromHost(ctx context.Context, host string) string { - if host == "" { - if s := os.Getenv("GAE_SERVICE"); s != "" { - return s - } - return "default" - } - - if idx := strings.Index(host, ":"); idx != -1 { - host = host[:idx] - } - - project := appengine.AppID(ctx) - if idx := strings.Index(project, "~"); idx != -1 { - project = project[idx+1:] - } - - pIdx := strings.Index(host, project) - if pIdx == -1 { - defaultHost := appengine.DefaultVersionHostname(ctx) - if host == defaultHost { - return "default" - } - return host - } - - domainSuffix := host[pIdx:] - if host == domainSuffix { - return "default" - } - - suffixes := []string{ - "." + domainSuffix, - "-dot-" + domainSuffix, - } - stripped := host - for _, suffix := range suffixes { - if strings.HasSuffix(stripped, suffix) { - stripped = stripped[:len(stripped)-len(suffix)] - break - } - } - - if stripped == host { - return host - } - - stripped = strings.ReplaceAll(stripped, "-dot-", ".") - parts := strings.Split(stripped, ".") - if len(parts) > 0 { - return parts[len(parts)-1] - } - return "default" -} - -func buildCloudTaskProto(ctx context.Context, queueName string, task *Task) (*taskspb.Task, string, error) { - if task.Name != "" { - if !taskNameRegex.MatchString(task.Name) { - return nil, "", fmt.Errorf("taskqueue: invalid task name %q", task.Name) - } - } - - if len(task.Payload) > maxTaskPayloadBytes { - return nil, "", fmt.Errorf("taskqueue: task too large (%d bytes)", len(task.Payload)) - } - - queuePath, err := getQueuePath(ctx, queueName) - if err != nil { - return nil, "", err - } - - taskName := task.Name - var fullTaskName string - if taskName != "" { - fullTaskName = fmt.Sprintf("%s/tasks/%s", queuePath, taskName) - } - - path := task.Path - if path == "" { - path = "/_ah/queue/" + queueName - } - - headers := make(map[string]string) - for k, vs := range task.Header { - if len(vs) > 0 { - headers[k] = vs[0] - } - } - - if _, ok := headers["Content-Type"]; !ok { - headers["Content-Type"] = "application/octet-stream" - } - if _, ok := headers["X-AppEngine-QueueName"]; !ok { - headers["X-AppEngine-QueueName"] = queueName - } - if taskName != "" { - if _, ok := headers["X-AppEngine-TaskName"]; !ok { - headers["X-AppEngine-TaskName"] = taskName - } - } - - targetService := extractServiceFromHost(ctx, headers["Host"]) - var routing *taskspb.AppEngineRouting - if targetService != "" { - routing = &taskspb.AppEngineRouting{ - Service: targetService, - } - } - delete(headers, "Host") - ae := &taskspb.AppEngineHttpRequest{ - RelativeUri: path, - Headers: headers, - Body: task.Payload, - AppEngineRouting: routing, - } - if code, ok := taskspb.HttpMethod_value[task.method()]; ok { - ae.HttpMethod = taskspb.HttpMethod(code) - } - - taskObj := &taskspb.Task{ - Name: fullTaskName, - PayloadType: &taskspb.Task_AppEngineHttpRequest{ - AppEngineHttpRequest: ae, - }, - } - - if !task.ETA.IsZero() { - taskObj.ScheduleTime = timestamppb.New(task.ETA) - } else if task.Delay > 0 { - taskObj.ScheduleTime = timestamppb.New(time.Now().Add(task.Delay)) - } - - if task.RetryOptions != nil { - rc := &taskspb.RetryConfig{} - hasRC := false - if task.RetryOptions.RetryLimit > 0 { - rc.MaxAttempts = task.RetryOptions.RetryLimit - hasRC = true - } - if task.RetryOptions.AgeLimit > 0 { - rc.MaxRetryDuration = durationpb.New(task.RetryOptions.AgeLimit) - hasRC = true - } - if task.RetryOptions.MinBackoff > 0 { - rc.MinBackoff = durationpb.New(task.RetryOptions.MinBackoff) - hasRC = true - } - if task.RetryOptions.MaxBackoff > 0 { - rc.MaxBackoff = durationpb.New(task.RetryOptions.MaxBackoff) - hasRC = true - } - if task.RetryOptions.MaxDoublings > 0 || (task.RetryOptions.MaxDoublings == 0 && task.RetryOptions.ApplyZeroMaxDoublings) { - rc.MaxDoublings = task.RetryOptions.MaxDoublings - hasRC = true - } - if hasRC { - taskObj.RetryConfig = rc - } - } - - return taskObj, taskName, nil -} - -func addInCloudTasks(ctx context.Context, task *Task, queueName string) (*Task, error) { - if queueName == "" { - queueName = "default" - } - - taskObj, taskName, err := buildCloudTaskProto(ctx, queueName, task) - if err != nil { - return nil, err - } - - // In App Engine Datastore, external HTTP/gRPC Cloud Tasks RPCs cannot participate - // in Datastore 2PC transactions. If we are running inside an active Datastore transaction, - // we stage the task as a _AE_PendingCloudTask entity in Datastore under the transaction. - // When the transaction commits, PostCommitHook dispatches the staged task to Cloud Tasks. - if t := internal.TransactionFromContext(ctx); t != nil { - handle := t.GetHandle() - pendingTasksMu.Lock() - if len(pendingTasks[handle]) >= maxTransactionalTasks { - pendingTasksMu.Unlock() - return nil, ErrTooManyTasksInTransaction - } - pendingTasksMu.Unlock() - - protoBytes, err := proto.Marshal(taskObj) - if err != nil { - return nil, fmt.Errorf("failed to marshal proto for transactional task: %v", err) - } - key := datastore.NewIncompleteKey(ctx, "_AE_PendingCloudTask", nil) - pendingTask := &PendingCloudTask{ - QueueName: queueName, - CloudTaskName: taskName, - CloudTaskPayload: string(protoBytes), - Created: time.Now(), - Status: "PENDING", - RetryCount: 0, - LastError: "", - HandledBySweeper: false, - SdkLang: "GO", - } - key, err = datastore.Put(ctx, key, pendingTask) - if err != nil { - return nil, fmt.Errorf("failed to save transactional task to Datastore: %v", err) - } - - pendingTasksMu.Lock() - pendingTasks[handle] = append(pendingTasks[handle], key.Encode()) - pendingTasksMu.Unlock() - - resultTask := *task - resultTask.Name = taskName - resultTask.Method = task.method() - return &resultTask, nil - } - - assignedName, err := sendTask(ctx, queueName, taskName, taskObj) - if err != nil { - return nil, err - } - - resultTask := *task - resultTask.Name = assignedName - resultTask.Method = task.method() - return &resultTask, nil -} - -func addMultiInCloudTasks(ctx context.Context, tasks []*Task, queueName string) ([]*Task, error) { - // If AddMulti is called inside a Datastore transaction, each task in the batch - // is transactionally staged in Datastore via addInCloudTasks so that all tasks - // commit atomically with the Datastore transaction. - if t := internal.TransactionFromContext(ctx); t != nil { - handle := t.GetHandle() - pendingTasksMu.Lock() - if len(pendingTasks[handle])+len(tasks) > maxTransactionalTasks { - pendingTasksMu.Unlock() - return nil, ErrTooManyTasksInTransaction - } - pendingTasksMu.Unlock() - - me, any := make(appengine.MultiError, len(tasks)), false - results := make([]*Task, len(tasks)) - for i, task := range tasks { - res, err := addInCloudTasks(ctx, task, queueName) - if err != nil { - me[i] = err - any = true - } else { - results[i] = res - } - } - if any { - return results, me - } - return results, nil - } - - fullQueueName, err := getQueuePath(ctx, queueName) - if err != nil { - return nil, err - } - - me, any := make(appengine.MultiError, len(tasks)), false - results := make([]*Task, len(tasks)) - - client, err := cloudtasks.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("failed to create cloudtasks client: %v", err) - } - defer client.Close() - - chunkSize := batchCreateChunkSize - for chunkStart := 0; chunkStart < len(tasks); chunkStart += chunkSize { - chunkEnd := chunkStart + chunkSize - if chunkEnd > len(tasks) { - chunkEnd = len(tasks) - } - chunkTasks := tasks[chunkStart:chunkEnd] - - createReqs := make([]*taskspb.CreateTaskRequest, 0, len(chunkTasks)) - for i, t := range chunkTasks { - taskObj, taskName, err := buildCloudTaskProto(ctx, queueName, t) - if err != nil { - me[chunkStart+i] = err - any = true - continue - } - results[chunkStart+i] = new(Task) - *results[chunkStart+i] = *t - results[chunkStart+i].Name = taskName - results[chunkStart+i].Method = t.method() - - createReqs = append(createReqs, &taskspb.CreateTaskRequest{ - Parent: fullQueueName, - Task: taskObj, - }) - } - if len(createReqs) == 0 { - continue - } - - batchReq := &taskspb.BatchCreateTasksRequest{ - Parent: fullQueueName, - Requests: createReqs, - } - - op, err := client.BatchCreateTasks(ctx, batchReq) - if err != nil { - if isUnimplementedError(err) { - for i, t := range chunkTasks { - if me[chunkStart+i] != nil { - continue - } - res, err := addInCloudTasks(ctx, t, queueName) - if err != nil { - me[chunkStart+i] = err - any = true - } else { - results[chunkStart+i] = res - } - } - } else { - for i := range chunkTasks { - if me[chunkStart+i] == nil { - me[chunkStart+i] = err - any = true - } - } - } - } else if op != nil { - meta, _ := op.Metadata() - resp, _ := op.Wait(ctx) - for i := range chunkTasks { - if meta != nil && meta.FailedRequests != nil { - if st, failed := meta.FailedRequests[int32(i)]; failed && st != nil && st.Code != 0 { - me[chunkStart+i] = mapOperationErrorCode(int(st.Code), st.Message, false) - any = true - continue - } - } - if resp != nil && i < len(resp.Tasks) && resp.Tasks[i] != nil { - createdTask := resp.Tasks[i] - if createdTask.Name != "" && results[chunkStart+i] != nil { - if idx := strings.LastIndex(createdTask.Name, "/"); idx != -1 { - results[chunkStart+i].Name = createdTask.Name[idx+1:] - } else { - results[chunkStart+i].Name = createdTask.Name - } - } - } - } - } - } - - if any { - return results, me - } - return results, nil -} - -func deleteMultiInCloudTasks(ctx context.Context, tasks []*Task, queueName string) error { - fullQueueName, err := getQueuePath(ctx, queueName) - if err != nil { - return err - } - - client, err := cloudtasks.NewClient(ctx) - if err != nil { - return fmt.Errorf("failed to create cloudtasks client: %v", err) - } - defer client.Close() - - me, any := make(appengine.MultiError, len(tasks)), false - - chunkSize := batchDeleteChunkSize - for chunkStart := 0; chunkStart < len(tasks); chunkStart += chunkSize { - chunkEnd := chunkStart + chunkSize - if chunkEnd > len(tasks) { - chunkEnd = len(tasks) - } - chunkTasks := tasks[chunkStart:chunkEnd] - - names := make([]string, len(chunkTasks)) - for i, t := range chunkTasks { - names[i] = fmt.Sprintf("%s/tasks/%s", fullQueueName, t.Name) - } - - batchReq := &taskspb.BatchDeleteTasksRequest{ - Parent: fullQueueName, - Names: names, - } - - op, err := client.BatchDeleteTasks(ctx, batchReq) - if err != nil { - for i := range chunkTasks { - me[chunkStart+i] = err - any = true - } - } else if op != nil { - meta, _ := op.Metadata() - for i := range chunkTasks { - if meta != nil && meta.FailedRequests != nil { - if st, failed := meta.FailedRequests[int32(i)]; failed && st != nil && st.Code != 0 { - me[chunkStart+i] = mapOperationErrorCode(int(st.Code), st.Message, true) - any = true - } - } - } - } - } - - if any { - return me - } - return nil -} - - - -func mapOperationErrorCode(code int, msg string, isDelete bool) error { - lowerMsg := strings.ToLower(msg) - isNotFound := code == grpcNotFound || code == httpNotFound || strings.Contains(lowerMsg, "not found") || strings.Contains(lowerMsg, "unknown") - isAlreadyExists := code == grpcAlreadyExists || code == httpAlreadyExists || strings.Contains(lowerMsg, "already exists") - - if isDelete && isNotFound { - return newUnknownTaskError(msg) - } - if isAlreadyExists || (isNotFound && strings.Contains(lowerMsg, "requested entity was not found")) { - return ErrTaskAlreadyAdded - } - if isNotFound { - return newUnknownTaskError(msg) - } - return fmt.Errorf("cloud tasks operation failed (%d): %s", code, msg) -} diff --git a/taskqueue/cloudtask_transactional.go b/taskqueue/cloudtask_transactional.go deleted file mode 100644 index f52c310a..00000000 --- a/taskqueue/cloudtask_transactional.go +++ /dev/null @@ -1,242 +0,0 @@ -package taskqueue - -import ( - "context" - "fmt" - "log" - "net/http" - "strings" - "sync" - "time" - - "google.golang.org/appengine" - "google.golang.org/appengine/datastore" - "google.golang.org/appengine/internal" - "google.golang.org/protobuf/proto" - - taskspb "cloud.google.com/go/cloudtasks/apiv2beta3/cloudtaskspb" -) - -const ( - statusPending = "PENDING" - statusProcessing = "PROCESSING" - statusFailed = "FAILED" - statusDone = "DONE" - statusAlreadyExists = "ALREADY_EXISTS" - - lockDuration = 60 * time.Second - fastPathGracePeriod = 60 * time.Second - maxSweeperRetries = 5 - maxLastErrorLength = 500 -) - -type PendingCloudTask struct { - QueueName string `datastore:"queue_name"` - CloudTaskName string `datastore:"cloud_task_name"` - CloudTaskPayload string `datastore:"cloud_task_payload,noindex"` - Created time.Time `datastore:"created"` - Status string `datastore:"status"` - LockExpires time.Time `datastore:"lock_expires"` - RetryCount int64 `datastore:"retry_count"` - LastError string `datastore:"last_error,noindex"` - HandledBySweeper bool `datastore:"handled_by_sweeper"` - SdkLang string `datastore:"sdk_lang"` -} - -var ( - pendingTasksMu sync.Mutex - pendingTasks = make(map[uint64][]string) // transaction handle -> list of urlsafe keys -) - -func init() { - internal.PostCommitHook = func(ctx context.Context, handle uint64) { - go dispatchPendingTasks(ctx, handle) - } - internal.RollbackHook = func(handle uint64) { - cleanupPendingTasks(handle) - } - http.HandleFunc("/_ah/cloudtask/sweep", handleSweep) -} - -func cleanupPendingTasks(handle uint64) { - pendingTasksMu.Lock() - delete(pendingTasks, handle) - pendingTasksMu.Unlock() -} - -type noCancelContext struct { - context.Context -} - -func (c *noCancelContext) Deadline() (deadline time.Time, ok bool) { - return time.Time{}, false -} - -func (c *noCancelContext) Done() <-chan struct{} { - return nil -} - -func (c *noCancelContext) Err() error { - return nil -} - -func logErrorf(ctx context.Context, format string, v ...interface{}) { - log.Printf("ERROR: "+format, v...) -} - -func dispatchPendingTasks(ctx context.Context, handle uint64) { - pendingTasksMu.Lock() - urlsafeKeys, ok := pendingTasks[handle] - if ok { - delete(pendingTasks, handle) - } - pendingTasksMu.Unlock() - - if !ok || len(urlsafeKeys) == 0 { - return - } - - noCancelCtx := &noCancelContext{Context: internal.TransactionlessContext(ctx)} - - for _, urlsafeKey := range urlsafeKeys { - key, err := datastore.DecodeKey(urlsafeKey) - if err != nil { - logErrorf(ctx, "Failed to decode pending task key: %v", err) - continue - } - - var taskEntity PendingCloudTask - err = datastore.Get(noCancelCtx, key, &taskEntity) - if err != nil { - logErrorf(ctx, "Failed to get pending task from Datastore: %v", err) - continue - } - - now := time.Now() - taskEntity.Status = statusProcessing - taskEntity.LockExpires = now.Add(lockDuration) - taskEntity.HandledBySweeper = false - if _, err := datastore.Put(noCancelCtx, key, &taskEntity); err != nil { - logErrorf(ctx, "Failed to acquire lock in fast-path for task %s: %v", taskEntity.CloudTaskName, err) - continue - } - - var taskObj taskspb.Task - if err := proto.Unmarshal([]byte(taskEntity.CloudTaskPayload), &taskObj); err != nil { - logErrorf(ctx, "Failed to unmarshal pending task proto: %v", err) - continue - } - _, err = sendTask(noCancelCtx, taskEntity.QueueName, taskEntity.CloudTaskName, &taskObj) - if err != nil { - if err == ErrTaskAlreadyAdded { - datastore.Delete(noCancelCtx, key) - continue - } - logErrorf(ctx, "Failed to dispatch task %s to queue %s: %v", taskEntity.CloudTaskName, taskEntity.QueueName, err) - taskEntity.RetryCount++ - taskEntity.LastError = err.Error() - if len(taskEntity.LastError) > maxLastErrorLength { - taskEntity.LastError = taskEntity.LastError[:maxLastErrorLength] - } - taskEntity.Status = statusPending - datastore.Put(noCancelCtx, key, &taskEntity) - continue - } - - err = datastore.Delete(noCancelCtx, key) - if err != nil { - logErrorf(ctx, "Failed to delete pending task %s from Datastore: %v", taskEntity.CloudTaskName, err) - } - } -} - -func sweep(ctx context.Context) error { - query := datastore.NewQuery("_AE_PendingCloudTask") - var tasks []PendingCloudTask - keys, err := query.GetAll(ctx, &tasks) - if err != nil { - return fmt.Errorf("failed to query _AE_PendingCloudTask: %v", err) - } - - now := time.Now() - count := 0 - for i, key := range keys { - task := tasks[i] - if task.Status == statusDone || task.Status == statusAlreadyExists { - continue - } - if task.Status == statusProcessing { - if !task.LockExpires.IsZero() && now.Before(task.LockExpires) { - continue // Still actively processing and lock valid - } else if task.LockExpires.IsZero() { - continue // Assume lock valid if just started - } - } else if task.Status == statusPending || task.Status == "" { - if !task.Created.IsZero() && now.Sub(task.Created) < fastPathGracePeriod { - continue // Give fast-path grace period to dispatch post-commit - } - } else if task.Status == statusFailed && task.RetryCount >= maxSweeperRetries { - continue // Exceeded max sweeper retries - } - - // Acquire lock - task.Status = statusProcessing - task.LockExpires = now.Add(lockDuration) - task.HandledBySweeper = true - if _, err := datastore.Put(ctx, key, &task); err != nil { - logErrorf(ctx, "Sweeper failed to acquire lock for task %s: %v", task.CloudTaskName, err) - continue - } - - var taskObj taskspb.Task - if err := proto.Unmarshal([]byte(task.CloudTaskPayload), &taskObj); err != nil { - logErrorf(ctx, "Sweeper failed to unmarshal pending task proto: %v", err) - continue - } - _, err := sendTask(ctx, task.QueueName, task.CloudTaskName, &taskObj) - if err != nil && err != ErrTaskAlreadyAdded { - logErrorf(ctx, "Sweeper failed to dispatch task %s: %v", task.CloudTaskName, err) - task.RetryCount++ - task.LastError = err.Error() - if len(task.LastError) > maxLastErrorLength { - task.LastError = task.LastError[:maxLastErrorLength] - } - if task.RetryCount >= maxSweeperRetries { - task.Status = statusFailed - task.LockExpires = time.Time{} - } else { - task.Status = statusPending - task.LockExpires = time.Time{} - } - if _, putErr := datastore.Put(ctx, key, &task); putErr != nil { - logErrorf(ctx, "Sweeper failed to record error state for task %s: %v", task.CloudTaskName, putErr) - } - continue - } - - if err := datastore.Delete(ctx, key); err != nil { - logErrorf(ctx, "Sweeper failed to delete entity %s: %v", task.CloudTaskName, err) - } - count++ - } - - log.Printf("Cloud Tasks sweeper processed %d tasks.", count) - return nil -} - -func handleSweep(w http.ResponseWriter, r *http.Request) { - isCron := strings.EqualFold(r.Header.Get("X-AppEngine-Cron"), "true") || strings.EqualFold(r.Header.Get("X-Appengine-Cron"), "true") - if !isCron && !appengine.IsDevAppServer() { - http.Error(w, "Access denied: endpoint only accessible via App Engine Cron.", http.StatusForbidden) - return - } - ctx := appengine.NewContext(r) - if err := sweep(ctx); err != nil { - logErrorf(ctx, "Sweeper failed: %v", err) - http.Error(w, fmt.Sprintf("Sweeper failed: %v", err), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - w.Write([]byte("Sweeper completed successfully.\n")) -} diff --git a/taskqueue/taskqueue.go b/taskqueue/taskqueue.go index ff09e1ef..4bbe7316 100644 --- a/taskqueue/taskqueue.go +++ b/taskqueue/taskqueue.go @@ -283,9 +283,6 @@ var alreadyAddedErrors = map[pb.TaskQueueServiceError_ErrorCode]bool{ // Add returns an equivalent Task with defaults filled in, including setting // the task's Name field to the chosen name if the original was empty. func Add(c context.Context, task *Task, queueName string) (*Task, error) { - if useCloudTasks() && task.Method != "PULL" { - return addInCloudTasks(c, task, queueName) - } req, err := newAddReq(c, task, queueName) if err != nil { return nil, err @@ -312,11 +309,6 @@ func Add(c context.Context, task *Task, queueName string) (*Task, error) { // each task's Name field to the chosen name if the original was empty. // If a given task is badly formed or could not be added, an appengine.MultiError is returned. func AddMulti(c context.Context, tasks []*Task, queueName string) ([]*Task, error) { - if useCloudTasks() { - if len(tasks) > 0 && tasks[0].Method != "PULL" { - return addMultiInCloudTasks(c, tasks, queueName) - } - } req := &pb.TaskQueueBulkAddRequest{ AddRequest: make([]*pb.TaskQueueAddRequest, len(tasks)), } @@ -375,11 +367,6 @@ func Delete(c context.Context, task *Task, queueName string) error { // Each task is deleted independently; one may fail to delete while the others // are successfully deleted. func DeleteMulti(c context.Context, tasks []*Task, queueName string) error { - if useCloudTasks() { - if len(tasks) > 0 && tasks[0].Method != "PULL" { - return deleteMultiInCloudTasks(c, tasks, queueName) - } - } taskNames := make([][]byte, len(tasks)) for i, t := range tasks { taskNames[i] = []byte(t.Name)