diff --git a/jobs/batch-permit-validator/Dockerfile b/jobs/batch-permit-validator/Dockerfile index 8e5ce2b5f..45c5587ab 100644 --- a/jobs/batch-permit-validator/Dockerfile +++ b/jobs/batch-permit-validator/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.2-bullseye AS development_build +FROM python:3.12-bookworm AS development_build USER root diff --git a/jobs/batch-permit-validator/devops/vaults.gcp.env b/jobs/batch-permit-validator/devops/vaults.gcp.env index a4330c7db..7a8add1e5 100644 --- a/jobs/batch-permit-validator/devops/vaults.gcp.env +++ b/jobs/batch-permit-validator/devops/vaults.gcp.env @@ -5,7 +5,6 @@ DATABASE_PORT="op://database/$APP_ENV/strr-db/DATABASE_PORT" DATABASE_UNIX_SOCKET="op://database/$APP_ENV/strr-db/DATABASE_UNIX_SOCKET" GCP_CS_PROJECT_ID="op://buckets/$APP_ENV/strr/GCP_CS_PROJECT_ID" GCP_CS_SA_SCOPE="op://buckets/$APP_ENV/strr/GCP_CS_SA_SCOPE" -GCP_AUTH_KEY="op://buckets/$APP_ENV/strr/GCP_AUTH_KEY" STR_DATA_API_CLIENT_ID="op://keycloak/$APP_ENV/str-data-api/STR_DATA_API_CLIENT_ID" STR_DATA_API_CLIENT_SECRET="op://keycloak/$APP_ENV/str-data-api/STR_DATA_API_CLIENT_SECRET" STR_DATA_API_TOKEN_URL="op://keycloak/$APP_ENV/str-data-api/STR_DATA_API_TOKEN_URL" diff --git a/jobs/batch-permit-validator/poetry.lock b/jobs/batch-permit-validator/poetry.lock index fcfb5506d..6cf57ff23 100644 --- a/jobs/batch-permit-validator/poetry.lock +++ b/jobs/batch-permit-validator/poetry.lock @@ -3687,7 +3687,7 @@ files = [ [[package]] name = "strr-api" -version = "0.3.29" +version = "0.3.33" description = "" optional = false python-versions = "^3.12.2" @@ -3727,8 +3727,8 @@ weasyprint = "^62.3" [package.source] type = "git" url = "https://github.com/bcgov/STRR.git" -reference = "main" -resolved_reference = "e2c2f26d5a30bc60d2477dbca1754bdf14a4b56b" +reference = "b3dd57ce3c1a4cb7f0694b5e06746302778a4433" +resolved_reference = "b3dd57ce3c1a4cb7f0694b5e06746302778a4433" subdirectory = "strr-api" [[package]] @@ -3736,14 +3736,14 @@ name = "strr-test-utils" version = "0.1.0" description = "Shared Test Utilities for STRR Jobs" optional = false -python-versions = "^3.12" +python-versions = ">=3.12.2,<4.0" groups = ["dev"] files = [] develop = true [package.dependencies] flask-sqlalchemy = "*" -pytest = "^8.0.0" +pytest = ">=8.0.0,<10.0.0" sqlalchemy = "*" structlog = "*" testcontainers = {version = "^4.0.0", extras = ["postgres", "redis"]} @@ -4260,4 +4260,4 @@ test = ["pytest"] [metadata] lock-version = "2.1" python-versions = "^3.12.2" -content-hash = "5f8e2f667868fde9e522a0a7bf27fc90cb03c69ff4abcd7f9b069f4322ee36ec" +content-hash = "29499c514e2f8ef067957bf4beafbd3374768cfd61c75cee6ac1dffbde31a817" diff --git a/jobs/batch-permit-validator/pyproject.toml b/jobs/batch-permit-validator/pyproject.toml index 5a264b3b7..0455188c9 100644 --- a/jobs/batch-permit-validator/pyproject.toml +++ b/jobs/batch-permit-validator/pyproject.toml @@ -21,7 +21,7 @@ pg8000 = "^1.31.2" gcp-queue = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/gcp-queue", branch = "main" } structured-logging = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/structured-logging", branch = "main" } # strr-api = {git = "https://github.com/bcgov/STRR.git", branch = "feature-validator-updates", subdirectory = "strr-api"} -strr-api = {git = "https://github.com/bcgov/STRR.git", rev = "main", subdirectory = "strr-api"} +strr-api = {git = "https://github.com/bcgov/STRR.git", rev = "b3dd57ce3c1a4cb7f0694b5e06746302778a4433", subdirectory = "strr-api"} nanoid = "^2.0.0" redis = "^7.0.0" diff --git a/jobs/batch-permit-validator/tests/unit/job/test_keyless_storage.py b/jobs/batch-permit-validator/tests/unit/job/test_keyless_storage.py new file mode 100644 index 000000000..f14f4b055 --- /dev/null +++ b/jobs/batch-permit-validator/tests/unit/job/test_keyless_storage.py @@ -0,0 +1,41 @@ +import base64 +from unittest.mock import patch + +from google.auth.credentials import Credentials +from google.cloud import storage + +from batch_permit_validator.job import GCPStorageService + + +class RuntimeCredentials(Credentials): + """Token-only credentials, like the job receives from its runtime identity.""" + + service_account_email = "sa-job@example.iam.gserviceaccount.com" + + def __init__(self): + super().__init__() + self.token = "synthetic-access-token" + + def refresh(self, request): + raise AssertionError("The synthetic token is already valid") + + +def test_installed_api_can_sign_response_url_with_runtime_credentials(): + """The job's installed API dependency must support IAM signing without a key.""" + credentials = RuntimeCredentials() + client = storage.Client(project="synthetic-project", credentials=credentials) + bucket = client.bucket("synthetic-response-bucket") + + with ( + patch.object(GCPStorageService, "get_bucket", return_value=bucket), + patch( + "google.cloud.storage._signing._sign_message", + return_value=base64.b64encode(b"synthetic-signature"), + ) as signer, + ): + url = GCPStorageService.get_presigned_url(bucket.name, "response.json", 10) + + assert "X-Goog-Signature=" in url + assert signer.call_count == 1 + assert signer.call_args.args[1] == credentials.token + assert signer.call_args.args[2] == credentials.service_account_email diff --git a/strr-api/src/strr_api/services/gcp_storage_service.py b/strr-api/src/strr_api/services/gcp_storage_service.py index 9603dfba9..6a72d9d13 100644 --- a/strr-api/src/strr_api/services/gcp_storage_service.py +++ b/strr-api/src/strr_api/services/gcp_storage_service.py @@ -40,6 +40,7 @@ from datetime import timedelta from flask import current_app, has_app_context +from google.auth.transport.requests import Request from google.cloud import storage from google.oauth2 import service_account @@ -192,9 +193,27 @@ def get_presigned_url(cls, bucket_id, blob_name, expiration_minutes): """Gets the presigned url for a file.""" bucket = cls.get_bucket(bucket_id) blob = bucket.blob(blob_name) + signed_url_kwargs = {} + + credentials = bucket.client._credentials # pylint: disable=protected-access + if credentials and not isinstance(credentials, service_account.Credentials): + # IAM signBlob is called outside the storage client's authenticated transport, + # so make sure ADC has a current access token before generating the URL. + if not credentials.valid: + credentials.refresh(Request()) + if service_account_email := getattr(credentials, "service_account_email", None): + signed_url_kwargs = { + "service_account_email": service_account_email, + "access_token": credentials.token, + } # Generate the signed URL - url = blob.generate_signed_url(version="v4", expiration=timedelta(minutes=expiration_minutes), method="GET") + url = blob.generate_signed_url( + version="v4", + expiration=timedelta(minutes=expiration_minutes), + method="GET", + **signed_url_kwargs, + ) return url diff --git a/strr-api/tests/unit/services/test_gcp_storage_service.py b/strr-api/tests/unit/services/test_gcp_storage_service.py index c0f0dff52..4a351bed0 100644 --- a/strr-api/tests/unit/services/test_gcp_storage_service.py +++ b/strr-api/tests/unit/services/test_gcp_storage_service.py @@ -187,6 +187,7 @@ def test_file_upload_and_presigned_url_succeed(mock_get_bucket, mock_uuid): mock_blob = MagicMock() mock_blob.generate_signed_url.return_value = "https://signed-url" mock_get_bucket.return_value.blob.return_value = mock_blob + mock_get_bucket.return_value.client.configure_mock(_credentials=None) upload_key = GCPStorageService.upload_file("text/csv", b"file contents", "target-bucket") url = GCPStorageService.get_presigned_url("target-bucket", "file-key", 10) @@ -194,7 +195,8 @@ def test_file_upload_and_presigned_url_succeed(mock_get_bucket, mock_uuid): assert upload_key == "file-key" assert url == "https://signed-url" mock_uuid.assert_called_once() - mock_get_bucket.assert_any_call("target-bucket") + assert mock_get_bucket.call_count == 2 + mock_get_bucket.assert_called_with("target-bucket") mock_get_bucket.return_value.blob.assert_any_call("file-key") mock_blob.upload_from_string.assert_called_once_with(data=b"file contents", content_type="text/csv") mock_blob.generate_signed_url.assert_called_once_with( @@ -204,6 +206,62 @@ def test_file_upload_and_presigned_url_succeed(mock_get_bucket, mock_uuid): ) +@patch("strr_api.services.gcp_storage_service.Request") +@patch("strr_api.services.gcp_storage_service.GCPStorageService.get_bucket") +def test_presigned_url_refreshes_invalid_adc_credentials(mock_get_bucket, mock_request): + """get_presigned_url supports keyless runtime credentials for signed URLs.""" + mock_credentials = MagicMock() + mock_credentials.valid = False + mock_credentials.service_account_email = "sa-job@example.iam.gserviceaccount.com" + mock_credentials.token = "access-token" + + mock_blob = MagicMock() + mock_blob.generate_signed_url.return_value = "https://signed-url" + mock_get_bucket.return_value.client.configure_mock(_credentials=mock_credentials) + mock_get_bucket.return_value.blob.return_value = mock_blob + + url = GCPStorageService.get_presigned_url("target-bucket", "file-key", 10) + + assert url == "https://signed-url" + mock_get_bucket.assert_called_once_with("target-bucket") + mock_credentials.refresh.assert_called_once_with(mock_request.return_value) + mock_blob.generate_signed_url.assert_called_once_with( + version="v4", + expiration=timedelta(minutes=10), + method="GET", + service_account_email="sa-job@example.iam.gserviceaccount.com", + access_token="access-token", + ) + + +@patch("strr_api.services.gcp_storage_service.Request") +@patch("strr_api.services.gcp_storage_service.GCPStorageService.get_bucket") +def test_presigned_url_reuses_valid_adc_credentials(mock_get_bucket, mock_request): + """get_presigned_url reuses a valid ADC access token.""" + mock_credentials = MagicMock() + mock_credentials.valid = True + mock_credentials.service_account_email = "sa-job@example.iam.gserviceaccount.com" + mock_credentials.token = "access-token" + + mock_blob = MagicMock() + mock_blob.generate_signed_url.return_value = "https://signed-url" + mock_get_bucket.return_value.client.configure_mock(_credentials=mock_credentials) + mock_get_bucket.return_value.blob.return_value = mock_blob + + url = GCPStorageService.get_presigned_url("target-bucket", "file-key", 10) + + assert url == "https://signed-url" + mock_credentials.refresh.assert_not_called() + mock_request.assert_not_called() + mock_blob.generate_signed_url.assert_called_once_with( + version="v4", + expiration=timedelta(minutes=10), + method="GET", + service_account_email="sa-job@example.iam.gserviceaccount.com", + access_token="access-token", + ) + + @patch("strr_api.services.gcp_storage_service.GCPStorageService.get_bucket") def test_upload_file_raises_external_service_exception_on_failure(mock_get_bucket): """upload_file wraps storage errors in ExternalServiceException."""