Migration to Pydantic v2: Enable compatibility with later FastAPI versions

CHANGELOG.md

@@ -15,6 +15,7 @@ ENHANCEMENTS:
 * Strengthen TRE API authentication with a layered `auth/` package (typed exceptions, `PyJWKClient`-backed token validation, immutable `AuthenticatedUser` model, composable RBAC factories), remove the redundant `AccessService` abstraction, and add Event Grid publish resilience with distinct Graph/publish failure reporting. ([#4989](https://github.com/microsoft/AzureTRE/pull/4989))
 * Add support for formatting UI code via `pre-commit` and fix existing formatting issues. ([#4955](https://github.com/microsoft/AzureTRE/issues/4955))
 * Update the version of `super-linter` used in the `build_validation_develop` workflow to 8.7.0 ([#4957](https://github.com/microsoft/AzureTRE/issues/4957))
+* Migration to Pydantic v2: Updates codebase to be compatible with Pydantic v2 for future FastAPI upgrades ([#4637](https://github.com/microsoft/AzureTRE/issues/4637))
 
 BUG FIXES:
 * Fix workspace deletion when backup is enabled for the base, unrestricted and airlock-import-review workspaces by adding a `delete_backups_on_uninstall` flag and a pre-teardown backup cleanup (`remove_backup.sh`) that stops protection and either deletes or retains the Recovery Services Vault, so deletion works with Azure secure-by-default soft delete ([#4962](https://github.com/microsoft/AzureTRE/issues/4962))

airlock_processor/StatusChangedQueueTrigger/__init__.py

@@ -10,13 +10,13 @@
 from exceptions import NoFilesInRequestException, TooManyFilesInRequestException
 
 from shared_code import blob_operations, constants
-from pydantic import BaseModel, parse_obj_as
+from pydantic import BaseModel, TypeAdapter
 
 
 class RequestProperties(BaseModel):
     request_id: str
     new_status: str
-    previous_status: Optional[str]
+    previous_status: Optional[str] = None
     type: str
     workspace_id: str
 
@@ -86,7 +86,7 @@ def extract_properties(msg: func.ServiceBusMessage) -> RequestProperties:
         body = msg.get_body().decode('utf-8')
         logging.debug('Python ServiceBus queue trigger processed message: %s', body)
         json_body = json.loads(body)
-        result = parse_obj_as(RequestProperties, json_body["data"])
+        result = TypeAdapter(RequestProperties).validate_python(json_body["data"])
         if not result:
             raise Exception("Failed parsing request properties")
     except json.decoder.JSONDecodeError:

airlock_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.8.11"
+__version__ = "0.8.12"

airlock_processor/requirements.txt

@@ -5,4 +5,4 @@ azure-storage-blob==12.27.1
 azure-identity==1.25.1
 azure-mgmt-storage==24.0.0
 azure-mgmt-resource==24.0.0
-pydantic==1.10.26
+pydantic==2.13.4

airlock_processor/tests/test_status_change_queue_trigger.py

@@ -20,6 +20,14 @@ def test_extract_prop_valid_body_return_all_values(self):
         assert req_prop.type == "101112"
         assert req_prop.workspace_id == "ws1"
 
+    def test_extract_prop_defaults_missing_previous_status_to_none(self):
+        message_body = "{ \"data\": { \"request_id\":\"123\",\"new_status\":\"draft\", \"type\":\"export\", \"workspace_id\":\"ws1\"  }}"
+        message = _mock_service_bus_message(body=message_body)
+
+        req_prop = extract_properties(message)
+
+        assert req_prop.previous_status is None
+
     def test_extract_prop_missing_arg_throws(self):
         message_body = "{ \"data\": { \"status\":\"456\" , \"type\":\"789\", \"workspace_id\":\"ws1\"  }}"
         message = _mock_service_bus_message(body=message_body)

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.26.0"
+__version__ = "0.26.1"

api_app/api/routes/resource_helpers.py

@@ -12,7 +12,7 @@
 from db.repositories.resources_history import ResourceHistoryRepository
 from models.domain.resource_template import ResourceTemplate
 from models.domain.authentication import User
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
 from db.errors import DuplicateEntity, EntityDoesNotExist
 from db.repositories.operations import OperationRepository
@@ -45,9 +45,9 @@ async def cascaded_update_resource(resource_patch: ResourcePatch, parent_resourc
         child_etag = child_resource["_etag"]
         primary_parent_service_name = ""
         if child_resource["resourceType"] == ResourceType.WorkspaceService:
-            child_resource = parse_obj_as(WorkspaceService, child_resource)
+            child_resource = TypeAdapter(WorkspaceService).validate_python(child_resource)
         elif child_resource["resourceType"] == ResourceType.UserResource:
-            child_resource = parse_obj_as(UserResource, child_resource)
+            child_resource = TypeAdapter(UserResource).validate_python(child_resource)
             primary_parent_workspace_service = await resource_repo.get_resource_by_id(child_resource.parentWorkspaceServiceId)
             primary_parent_service_name = primary_parent_workspace_service.templateName
 
@@ -65,7 +65,7 @@ async def save_and_deploy_resource(
     resource_template: ResourceTemplate,
 ) -> Operation:
     try:
-        resource.user = user
+        resource.user = user.model_dump()
         resource.updatedWhen = get_timestamp()
 
         # Making a copy to save with secrets masked
@@ -134,7 +134,7 @@ def flatten_template_props(template_fragment: dict):
             if isinstance(prop, dict) and prop_name != "if":
                 flatten_template_props(prop)
 
-    flatten_template_props(template.dict())
+    flatten_template_props(template.model_dump())
 
     def recurse_input_props(prop_dict: dict):
         for prop_name, prop in prop_dict.items():

api_app/api/routes/shared_service_templates.py

@@ -1,6 +1,6 @@
 from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, status
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
 from api.helpers import get_repository
 from db.errors import EntityDoesNotExist, EntityVersionExist, InvalidInput
@@ -26,7 +26,7 @@ async def get_shared_service_templates(authorized_only: bool = False, template_r
 async def get_shared_service_template(shared_service_template_name: str, is_update: bool = False, version: Optional[str] = None, template_repo=Depends(get_repository(ResourceTemplateRepository))) -> SharedServiceTemplateInResponse:
     try:
         template = await get_template(shared_service_template_name, template_repo, ResourceType.SharedService, is_update=is_update, version=version)
-        return parse_obj_as(SharedServiceTemplateInResponse, template)
+        return TypeAdapter(SharedServiceTemplateInResponse).validate_python(template)
     except EntityDoesNotExist:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=strings.SHARED_SERVICE_TEMPLATE_DOES_NOT_EXIST)
 

api_app/api/routes/shared_services.py

@@ -32,23 +32,23 @@ def user_is_tre_admin(user):
     return False
 
 
-@shared_services_router.get("/shared-services", response_model=SharedServicesInList, name=strings.API_GET_ALL_SHARED_SERVICES, dependencies=[Depends(require_tre_user_or_admin)])
-async def retrieve_shared_services(shared_services_repo=Depends(get_repository(SharedServiceRepository)), user=Depends(require_tre_user_or_admin), resource_template_repo=Depends(get_repository(ResourceTemplateRepository))) -> SharedServicesInList:
+@shared_services_router.get("/shared-services", response_model=SharedServicesInList | RestrictedSharedServicesInList, name=strings.API_GET_ALL_SHARED_SERVICES, dependencies=[Depends(require_tre_user_or_admin)])
+async def retrieve_shared_services(shared_services_repo=Depends(get_repository(SharedServiceRepository)), user=Depends(require_tre_user_or_admin), resource_template_repo=Depends(get_repository(ResourceTemplateRepository))) -> SharedServicesInList | RestrictedSharedServicesInList:
     shared_services = await shared_services_repo.get_active_shared_services()
     await asyncio.gather(*[enrich_resource_with_available_upgrades(shared_service, resource_template_repo) for shared_service in shared_services])
     if user_is_tre_admin(user):
         return SharedServicesInList(sharedServices=shared_services)
     else:
-        return RestrictedSharedServicesInList(sharedServices=shared_services)
+        return RestrictedSharedServicesInList(sharedServices=[service.model_dump() for service in shared_services])
 
 
-@shared_services_router.get("/shared-services/{shared_service_id}", response_model=SharedServiceInResponse, name=strings.API_GET_SHARED_SERVICE_BY_ID, dependencies=[Depends(require_tre_user_or_admin), Depends(get_shared_service_by_id_from_path)])
-async def retrieve_shared_service_by_id(shared_service=Depends(get_shared_service_by_id_from_path), user=Depends(require_tre_user_or_admin), resource_template_repo=Depends(get_repository(ResourceTemplateRepository))):
+@shared_services_router.get("/shared-services/{shared_service_id}", response_model=SharedServiceInResponse | RestrictedSharedServiceInResponse, name=strings.API_GET_SHARED_SERVICE_BY_ID, dependencies=[Depends(require_tre_user_or_admin), Depends(get_shared_service_by_id_from_path)])
+async def retrieve_shared_service_by_id(shared_service=Depends(get_shared_service_by_id_from_path), user=Depends(require_tre_user_or_admin), resource_template_repo=Depends(get_repository(ResourceTemplateRepository))) -> SharedServiceInResponse | RestrictedSharedServiceInResponse:
     await enrich_resource_with_available_upgrades(shared_service, resource_template_repo)
     if user_is_tre_admin(user):
         return SharedServiceInResponse(sharedService=shared_service)
     else:
-        return RestrictedSharedServiceInResponse(sharedService=shared_service)
+        return RestrictedSharedServiceInResponse(sharedService=shared_service.model_dump())
 
 
 @shared_services_router.post("/shared-services", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_CREATE_SHARED_SERVICE, dependencies=[Depends(require_tre_admin)])

api_app/api/routes/user_resource_templates.py

@@ -1,15 +1,15 @@
 
 from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, status
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
 from api.dependencies.workspace_service_templates import get_workspace_service_template_by_name_from_path
 from api.routes.resource_helpers import get_template
-from db.errors import EntityVersionExist, InvalidInput
 from api.helpers import get_repository
+from db.errors import EntityVersionExist, InvalidInput
 from db.repositories.resource_templates import ResourceTemplateRepository
 from models.domain.resource import ResourceType
-from models.schemas.user_resource_template import UserResourceTemplateInResponse, UserResourceTemplateInCreate
+from models.schemas.user_resource_template import UserResourceTemplateInCreate, UserResourceTemplateInResponse
 from models.schemas.resource_template import ResourceTemplateInformationInList
 from resources import strings
 from auth.rbac import require_tre_admin, require_tre_user_or_admin
@@ -27,7 +27,7 @@ async def get_user_resource_templates_for_service_template(service_template_name
 @user_resource_templates_core_router.get("/workspace-service-templates/{service_template_name}/user-resource-templates/{user_resource_template_name}", response_model=UserResourceTemplateInResponse, response_model_exclude_none=True, name=strings.API_GET_USER_RESOURCE_TEMPLATE_BY_NAME, dependencies=[Depends(require_tre_user_or_admin)])
 async def get_user_resource_template(service_template_name: str, user_resource_template_name: str, is_update: bool = False, version: Optional[str] = None, template_repo=Depends(get_repository(ResourceTemplateRepository))) -> UserResourceTemplateInResponse:
     template = await get_template(user_resource_template_name, template_repo, ResourceType.UserResource, service_template_name, is_update=is_update, version=version)
-    return parse_obj_as(UserResourceTemplateInResponse, template)
+    return TypeAdapter(UserResourceTemplateInResponse).validate_python(template)
 
 
 @user_resource_templates_core_router.post("/workspace-service-templates/{service_template_name}/user-resource-templates", status_code=status.HTTP_201_CREATED, response_model=UserResourceTemplateInResponse, response_model_exclude_none=True, name=strings.API_CREATE_USER_RESOURCE_TEMPLATES, dependencies=[Depends(require_tre_admin)])

api_app/api/routes/workspace_service_templates.py

@@ -1,10 +1,10 @@
 from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, status
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
 from api.routes.resource_helpers import get_template
-from db.errors import EntityVersionExist, InvalidInput
 from api.helpers import get_repository
+from db.errors import EntityVersionExist, InvalidInput
 from db.repositories.resource_templates import ResourceTemplateRepository
 from models.domain.resource import ResourceType
 from models.schemas.resource_template import ResourceTemplateInResponse, ResourceTemplateInformationInList
@@ -25,7 +25,7 @@ async def get_workspace_service_templates(template_repo=Depends(get_repository(R
 @workspace_service_templates_core_router.get("/workspace-service-templates/{service_template_name}", response_model=WorkspaceServiceTemplateInResponse, response_model_exclude_none=True, name=strings.API_GET_WORKSPACE_SERVICE_TEMPLATE_BY_NAME, dependencies=[Depends(require_tre_user_or_admin)])
 async def get_workspace_service_template(service_template_name: str, is_update: bool = False, version: Optional[str] = None, template_repo=Depends(get_repository(ResourceTemplateRepository))) -> WorkspaceServiceTemplateInResponse:
     template = await get_template(service_template_name, template_repo, ResourceType.WorkspaceService, is_update=is_update, version=version)
-    return parse_obj_as(WorkspaceServiceTemplateInResponse, template)
+    return TypeAdapter(WorkspaceServiceTemplateInResponse).validate_python(template)
 
 
 @workspace_service_templates_core_router.post("/workspace-service-templates", status_code=status.HTTP_201_CREATED, response_model=WorkspaceServiceTemplateInResponse, response_model_exclude_none=True, name=strings.API_CREATE_WORKSPACE_SERVICE_TEMPLATES, dependencies=[Depends(require_tre_admin)])

api_app/api/routes/workspace_templates.py

@@ -1,6 +1,6 @@
 from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, status
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
 from api.helpers import get_repository
 from db.errors import EntityVersionExist, InvalidInput
@@ -25,7 +25,7 @@ async def get_workspace_templates(authorized_only: bool = False, template_repo=D
 @workspace_templates_admin_router.get("/workspace-templates/{workspace_template_name}", response_model=WorkspaceTemplateInResponse, name=strings.API_GET_WORKSPACE_TEMPLATE_BY_NAME, response_model_exclude_none=True)
 async def get_workspace_template(workspace_template_name: str, is_update: bool = False, version: Optional[str] = None, template_repo=Depends(get_repository(ResourceTemplateRepository))) -> WorkspaceTemplateInResponse:
     template = await get_template(workspace_template_name, template_repo, ResourceType.Workspace, is_update=is_update, version=version)
-    return parse_obj_as(WorkspaceTemplateInResponse, template)
+    return TypeAdapter(WorkspaceTemplateInResponse).validate_python(template)
 
 
 @workspace_templates_admin_router.post("/workspace-templates", status_code=status.HTTP_201_CREATED, response_model=WorkspaceTemplateInResponse, response_model_exclude_none=True, name=strings.API_CREATE_WORKSPACE_TEMPLATES)

api_app/auth/models.py

@@ -1,7 +1,7 @@
 from enum import StrEnum
 from typing import Optional, Tuple, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class TRERole(StrEnum):
@@ -25,16 +25,15 @@ class AuthenticatedUser(BaseModel):
     in place (e.g. ``roles.append(...)``) after creation.
     """
 
+    model_config = ConfigDict(frozen=True)
+
     id: str
     name: str
     email: Optional[str] = None
     roles: Tuple[str, ...] = Field(default_factory=tuple)
     audience: str = ""
     is_workspace_token: bool = False
 
-    class Config:
-        frozen = True
-
     def has_any_role(self, *roles: Union[TRERole, WorkspaceAccessRole]) -> bool:
         """Return *True* if the user holds at least one of *roles*."""
         role_values = {r.value for r in roles}

api_app/db/repositories/airlock_requests.py

@@ -2,11 +2,11 @@
 import uuid
 
 from datetime import datetime, timezone, UTC
-from typing import List, Optional
+from typing import List, Optional, Union
 from pydantic import UUID4
 from azure.cosmos.exceptions import CosmosResourceNotFoundError, CosmosAccessConditionFailedError
 from fastapi import HTTPException, status
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 from db.repositories.workspaces import WorkspaceRepository
 from services.authentication import get_aad_service
 from models.domain.authentication import User
@@ -34,7 +34,7 @@ def get_resource_base_spec_params():
     def get_timestamp(self) -> float:
         return datetime.now(timezone.utc).timestamp()
 
-    async def update_airlock_request_item(self, original_request: AirlockRequest, new_request: AirlockRequest, updated_by: User, request_properties: dict) -> AirlockRequest:
+    async def update_airlock_request_item(self, original_request: AirlockRequest, new_request: AirlockRequest, updated_by: Union[User, dict], request_properties: dict) -> AirlockRequest:
         history_item = AirlockRequestHistoryItem(
             resourceVersion=original_request.resourceVersion,
             updatedWhen=original_request.updatedWhen,
@@ -45,7 +45,12 @@ async def update_airlock_request_item(self, original_request: AirlockRequest, ne
 
         # now update the request props
         new_request.resourceVersion = new_request.resourceVersion + 1
-        new_request.updatedBy = updated_by
+        if hasattr(updated_by, "model_dump"):
+            new_request.updatedBy = updated_by.model_dump()
+        elif isinstance(updated_by, dict):
+            new_request.updatedBy = updated_by
+        else:
+            raise TypeError("updated_by must be a User model or dict")
         new_request.updatedWhen = self.get_timestamp()
 
         await self.upsert_item_with_etag(new_request, new_request.etag)
@@ -151,14 +156,14 @@ async def get_airlock_requests(self, workspace_id: Optional[str] = None, creator
             query += ' ASC' if order_ascending else ' DESC'
 
         airlock_requests = await self.query(query=query, parameters=parameters)
-        return parse_obj_as(List[AirlockRequest], airlock_requests)
+        return TypeAdapter(List[AirlockRequest]).validate_python(airlock_requests)
 
     async def get_airlock_request_by_id(self, airlock_request_id: UUID4) -> AirlockRequest:
         try:
             airlock_requests = await self.read_item_by_id(str(airlock_request_id))
         except CosmosResourceNotFoundError:
             raise EntityDoesNotExist
-        return parse_obj_as(AirlockRequest, airlock_requests)
+        return TypeAdapter(AirlockRequest).validate_python(airlock_requests)
 
     async def get_airlock_requests_for_airlock_manager(self, user_id: str, type: Optional[AirlockRequestType] = None, status: Optional[AirlockRequestStatus] = None, order_by: Optional[str] = None, order_ascending=True) -> List[AirlockRequest]:
         workspace_repo = await WorkspaceRepository.create()
@@ -184,7 +189,7 @@ async def get_airlock_requests_for_airlock_manager(self, user_id: str, type: Opt
     async def update_airlock_request(
             self,
             original_request: AirlockRequest,
-            updated_by: User,
+            updated_by: Union[User, dict],
             new_status: Optional[AirlockRequestStatus] = None,
             request_files: Optional[List[AirlockFile]] = None,
             status_message: Optional[str] = None,
@@ -246,7 +251,7 @@ def _build_updated_request(
             status_message: Optional[Optional[str]] = None,
             airlock_review: Optional[AirlockReview] = None,
             review_user_resource: Optional[AirlockReviewUserResource] = None,
-            updated_by: Optional[User] = None) -> AirlockRequest:
+            updated_by: Optional[Union[User, dict]] = None) -> AirlockRequest:
         updated_request = copy.deepcopy(original_request)
 
         if new_status is not None:
@@ -266,7 +271,9 @@ def _build_updated_request(
                 updated_request.reviews.append(airlock_review)
 
         if review_user_resource is not None and updated_by is not None:
-            updated_request.reviewUserResources[updated_by.id] = review_user_resource
+            reviewer_id = updated_by.id if hasattr(updated_by, "id") else updated_by.get("id")
+            if reviewer_id:
+                updated_request.reviewUserResources[reviewer_id] = review_user_resource
 
         return updated_request