From d099146b0e7dfa5278df40de48a6650a641abc3f Mon Sep 17 00:00:00 2001 From: chenm1 Date: Tue, 7 Jul 2026 20:52:58 -0700 Subject: [PATCH 1/7] fix: [SG-43950] Support component schemas in schema provisioning Issue ---- 1. Schema provisioning failed with HTTP 415 errors when querying existing schemas via `schemas_by_super_type`, which triggered cross-org-group enumeration on the backend. 2. Previous implementation of schema module only support `kind="type"` schemas, blocking creation of `kind="component"` schemas. Root Cause: ---- - `schemas_by_super_type` enumerated all org groups in the collection, causing HTTP 415 from unrelated org groups. - `_schema_tree` hardcoded component type IDs (BINARY, COMMENT, DER_SOURCE) as root nodes instead of inheriting from BASE_COMPONENT_TYPE_ID. - `cache_schema_config` always appended BASE_TYPE_ID as parent regardless of schema kind, making component schemas resolve ancestry incorrectly. - Schema creation raised an error if a schema already existed in the collection but not in our library (e.g. on first provisioning run). Fix Applied: ---- - Replace `schemas_by_super_type` with `schemas_by_library_id` scoped to `FLOW_TOOLKIT_LIBRARY_ID`, avoiding cross-org enumeration. - Handle library-not-found (permission error or NOT_FOUND) as empty set so first-run provisioning works correctly. - Add `BASE_COMPONENT_TYPE_ID`, `BASE_PROPERTY_TYPE_ID`, and `KIND_BASE_TYPE_ID` constants to `globals.py`. - Fix `_schema_tree` so component types correctly inherit from BASE_COMPONENT_TYPE_ID`. - Update `cache_schema_config` to use kind-appropriate base type and validate the `kind` field. - Add `kind` as a mandatory key in `SchemaBuilder` validation. - Catch "already exists" error in `build()` and treat it as success, allowing the version field to be updated and skipping re-provisioning on subsequent runs. --- .../flow_integration_sdk/globals.py | 9 ++- .../flow_integration_sdk/schema.py | 31 +++++++--- .../flow_integration_sdk/schema_builder.py | 61 ++++++++++++------- 3 files changed, 71 insertions(+), 30 deletions(-) diff --git a/python/tank_vendor/flow_integration_sdk/globals.py b/python/tank_vendor/flow_integration_sdk/globals.py index a86599ebc..39daa12a9 100644 --- a/python/tank_vendor/flow_integration_sdk/globals.py +++ b/python/tank_vendor/flow_integration_sdk/globals.py @@ -27,7 +27,7 @@ from .exceptions import FlowError from .utils import get_logger -# Component type ids +# Schema type ids # ------------------ # Type ids correspond to specific MEDM schemas (and versions). # Schemas can be created in a hierarchical fashion using inheritance. @@ -35,6 +35,13 @@ # commonly relevant to asset management. BASE_TYPE_ID = "autodesk.me:type-1.1.0" +BASE_COMPONENT_TYPE_ID = "autodesk.me:component-1.0.0" +BASE_PROPERTY_TYPE_ID = "autodesk.me:property-1.0.0" +KIND_BASE_TYPE_ID = { + "component": BASE_COMPONENT_TYPE_ID, + "property": BASE_PROPERTY_TYPE_ID, + "type": BASE_TYPE_ID, +} BINARY_TYPE_ID = "autodesk.me:component.binary-1.0.0" COMMENT_TYPE_ID = "autodesk.me:component.publishComment-1.0.0" # NOTE: This is a temporary schema being annexed for representing derivative source diff --git a/python/tank_vendor/flow_integration_sdk/schema.py b/python/tank_vendor/flow_integration_sdk/schema.py index 7675f1204..4b6188966 100644 --- a/python/tank_vendor/flow_integration_sdk/schema.py +++ b/python/tank_vendor/flow_integration_sdk/schema.py @@ -23,6 +23,8 @@ from .exceptions import FlowError from .globals import ( + BASE_COMPONENT_TYPE_ID, + BASE_PROPERTY_TYPE_ID, BASE_TYPE_ID, BINARY_TYPE_ID, COMMENT_TYPE_ID, @@ -31,6 +33,7 @@ get_client, get_session_collection, IMAGE_TYPE_ID, + KIND_BASE_TYPE_ID, ) from .utils import get_logger, trace @@ -43,12 +46,14 @@ _schema_tree: dict[str, list[str]] = {} # Hardcode some well known relationships and root types -_schema_tree[IMAGE_TYPE_ID] = [BINARY_TYPE_ID] -_schema_tree[FOLDER_TYPE_ID] = [BASE_TYPE_ID] +_schema_tree[BASE_COMPONENT_TYPE_ID] = [] +_schema_tree[BASE_PROPERTY_TYPE_ID] = [] _schema_tree[BASE_TYPE_ID] = [] -_schema_tree[BINARY_TYPE_ID] = [] -_schema_tree[COMMENT_TYPE_ID] = [] -_schema_tree[DER_SOURCE_TYPE_ID] = [] +_schema_tree[BINARY_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] +_schema_tree[COMMENT_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] +_schema_tree[DER_SOURCE_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] +_schema_tree[FOLDER_TYPE_ID] = [BASE_TYPE_ID] +_schema_tree[IMAGE_TYPE_ID] = [BINARY_TYPE_ID] # Schema type ids cache @@ -121,13 +126,23 @@ def cache_schema_config(config_path: str): # Add configured types to schema tree cache for schema in raw_config.get("schemas", {}): type_name = schema.get("name", "") + kind = schema.get("kind") + if not kind: + raise ValueError( + f"Schema '{type_name}' is missing required 'kind' field." + ) parent_types = schema.get("inherits", []) - # strip "$ref:" suffix + # strip "$ref:" prefix parent_types = [pt[5:] for pt in parent_types] # convert to full ids parent_types = [get_schema_id(pt) for pt in parent_types] - # always include base type - parent_types.append(BASE_TYPE_ID) + # always include the kind-appropriate base type + if kind not in KIND_BASE_TYPE_ID: + raise ValueError( + f"Unknown schema kind '{kind}' for '{type_name}'. " + f"Must be one of: {', '.join(KIND_BASE_TYPE_ID)}" + ) + parent_types.append(KIND_BASE_TYPE_ID[kind]) type_id = get_schema_id(type_name) # store ancestral relationship if type_id: diff --git a/python/tank_vendor/flow_integration_sdk/schema_builder.py b/python/tank_vendor/flow_integration_sdk/schema_builder.py index d468af814..336eb758e 100644 --- a/python/tank_vendor/flow_integration_sdk/schema_builder.py +++ b/python/tank_vendor/flow_integration_sdk/schema_builder.py @@ -27,11 +27,11 @@ FlowSchemaError, FlowSchemaLibraryError, ) -from .globals import BASE_TYPE_ID, get_client +from .globals import get_client from .objects import FlowProject from . import schema from .schema import get_schema_id -from .utils import cleanpath, get_logger +from .utils import get_logger # Constant for Flow Toolkit Library schema library ID @@ -90,10 +90,10 @@ def __validate_schema_dict(self) -> None: } # Check for mandatory keys - for key in ["name", "version"]: + for key in ["name", "version", "kind"]: if key not in self.schema_dict: raise KeyError( - f"The custom type schema must contain a '{key}' key of type string." + f"The schema definition must contain a '{key}' key of type string." ) # Validate keys and types @@ -296,8 +296,21 @@ def build(self) -> flow_model.Schema: ) query_response = create_schema_query.call() self.schema = query_response.schema - logger.info("Created new schema: %s", self.schema.name) - except (GQLAPIError, FlowConnectionError, ValidationError) as e: + logger.info(f"Created new schema: {self.schema.name}") + except GQLAPIError as e: + if "already exists" in str(e): + logger.info( + f"Schema '{self.schema_dict['name']}' version '{self.schema_dict['version']}' already exists in collection. " + "Skipping creation." + ) + else: + raise FlowSchemaError( + details=( + f"Failed to create schema '{self.schema_dict['name']}' " + f"version '{self.schema_dict['version']}': {e}" + ) + ) from e + except (FlowConnectionError, ValidationError) as e: raise FlowSchemaError( details=( f"Failed to create schema '{self.schema_dict['name']}' " @@ -546,24 +559,31 @@ def create_pipeline_schemas(project_id: str, config_path: str): client = get_client() collection_id = FlowProject.get_collection_id(project_id) - # Retrieve all existing published schema type ids for the collection + # Retrieve all existing schema type ids in our library. + # If the library does not exist yet (first run), treat as empty. try: - # Create SchemasBySuperTypeInput for the query - schemas_by_supertype_input = flow_model.SchemasBySuperTypeInput( - collection_id=collection_id, - type_id=BASE_TYPE_ID, - include_sub_sub_classes=True, + schemas_by_library_input = flow_model.SchemasByLibraryIdInput( + project_id=project_id, + library_id=FLOW_TOOLKIT_LIBRARY_ID, ) - # Create and call the schema query - schema_query = client.service_schema.schemas_by_super_type( - variables=schemas_by_supertype_input + schema_query = client.service_schema.schemas_by_library_id( + variables=schemas_by_library_input ) - existing_schema_types = set(schema_query.schema_types_iterator) + existing_schema_types = {s.type_id for s in schema_query.schemas_iterator} + logger.info( - f"Retrieved {len(existing_schema_types)} existing schemas for " - f"collection {collection_id}." + f"Retrieved {len(existing_schema_types)} existing schemas from " + f"library '{FLOW_TOOLKIT_LIBRARY_ID}'." ) - except (GQLAPIError, FlowConnectionError, ValidationError) as e: + except GQLAPIError as e: + if "NOT_FOUND" in str(e) or "do not have permission" in str(e): + logger.info( + f"Library '{FLOW_TOOLKIT_LIBRARY_ID}' not found. Assuming no existing schemas." + ) + existing_schema_types = set() + else: + raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e + except (FlowConnectionError, ValidationError) as e: raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e # Check if schema listed in config.json already exists @@ -574,8 +594,7 @@ def create_pipeline_schemas(project_id: str, config_path: str): schema_type_id = get_schema_id(schema_dict["name"]) if schema_type_id not in existing_schema_types: logger.info( - "Schema '%s' not found. Adding to list to be created.", - schema_type_id, + f"Schema '{schema_type_id}' not found. Adding to list to be created." ) need_to_create.append(schema_dict) From 7d830eebd3cb9215a51002b8be79d9297fb9affb Mon Sep 17 00:00:00 2001 From: chenm1 Date: Wed, 8 Jul 2026 13:38:42 -0700 Subject: [PATCH 2/7] Update schema query to use schemas_by_super_type again instead of schemas_by_library_id --- .../flow_integration_sdk/globals.py | 14 +++-- .../flow_integration_sdk/schema_builder.py | 56 +++++++------------ 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/python/tank_vendor/flow_integration_sdk/globals.py b/python/tank_vendor/flow_integration_sdk/globals.py index 39daa12a9..69b4d7994 100644 --- a/python/tank_vendor/flow_integration_sdk/globals.py +++ b/python/tank_vendor/flow_integration_sdk/globals.py @@ -34,14 +34,9 @@ # The schemas below are official Autodesk supported types that are # commonly relevant to asset management. -BASE_TYPE_ID = "autodesk.me:type-1.1.0" BASE_COMPONENT_TYPE_ID = "autodesk.me:component-1.0.0" BASE_PROPERTY_TYPE_ID = "autodesk.me:property-1.0.0" -KIND_BASE_TYPE_ID = { - "component": BASE_COMPONENT_TYPE_ID, - "property": BASE_PROPERTY_TYPE_ID, - "type": BASE_TYPE_ID, -} +BASE_TYPE_ID = "autodesk.me:type-1.1.0" BINARY_TYPE_ID = "autodesk.me:component.binary-1.0.0" COMMENT_TYPE_ID = "autodesk.me:component.publishComment-1.0.0" # NOTE: This is a temporary schema being annexed for representing derivative source @@ -50,6 +45,13 @@ FOLDER_TYPE_ID = "autodesk.me:type.folder-1.0.0" IMAGE_TYPE_ID = "autodesk.me:component.binary.image-1.0.0" +# Maps schema kind name to its root base type ID +KIND_BASE_TYPE_ID = { + "component": BASE_COMPONENT_TYPE_ID, + "property": BASE_PROPERTY_TYPE_ID, + "type": BASE_TYPE_ID, +} + # Component types # --------------- # Component base type names without full ids. diff --git a/python/tank_vendor/flow_integration_sdk/schema_builder.py b/python/tank_vendor/flow_integration_sdk/schema_builder.py index 336eb758e..637393da1 100644 --- a/python/tank_vendor/flow_integration_sdk/schema_builder.py +++ b/python/tank_vendor/flow_integration_sdk/schema_builder.py @@ -27,7 +27,7 @@ FlowSchemaError, FlowSchemaLibraryError, ) -from .globals import get_client +from .globals import KIND_BASE_TYPE_ID, get_client from .objects import FlowProject from . import schema from .schema import get_schema_id @@ -297,20 +297,7 @@ def build(self) -> flow_model.Schema: query_response = create_schema_query.call() self.schema = query_response.schema logger.info(f"Created new schema: {self.schema.name}") - except GQLAPIError as e: - if "already exists" in str(e): - logger.info( - f"Schema '{self.schema_dict['name']}' version '{self.schema_dict['version']}' already exists in collection. " - "Skipping creation." - ) - else: - raise FlowSchemaError( - details=( - f"Failed to create schema '{self.schema_dict['name']}' " - f"version '{self.schema_dict['version']}': {e}" - ) - ) from e - except (FlowConnectionError, ValidationError) as e: + except (GQLAPIError, FlowConnectionError, ValidationError) as e: raise FlowSchemaError( details=( f"Failed to create schema '{self.schema_dict['name']}' " @@ -559,30 +546,29 @@ def create_pipeline_schemas(project_id: str, config_path: str): client = get_client() collection_id = FlowProject.get_collection_id(project_id) - # Retrieve all existing schema type ids in our library. - # If the library does not exist yet (first run), treat as empty. - try: - schemas_by_library_input = flow_model.SchemasByLibraryIdInput( - project_id=project_id, - library_id=FLOW_TOOLKIT_LIBRARY_ID, - ) - schema_query = client.service_schema.schemas_by_library_id( - variables=schemas_by_library_input - ) - existing_schema_types = {s.type_id for s in schema_query.schemas_iterator} + # Collect distinct schema kinds present in config, then query existing + # schemas per kind. This avoids querying for kinds not used in the config. + kinds_in_config = {s["kind"] for s in config.get("schemas", []) if "kind" in s} + existing_schema_types = set() + try: + for kind in kinds_in_config: + base_type_id = KIND_BASE_TYPE_ID[kind] + schemas_by_supertype_input = flow_model.SchemasBySuperTypeInput( + collection_id=collection_id, + type_id=base_type_id, + include_sub_sub_classes=True, + ) + schema_query = client.service_schema.schemas_by_super_type( + variables=schemas_by_supertype_input + ) + existing_schema_types.update(schema_query.schema_types_iterator) logger.info( - f"Retrieved {len(existing_schema_types)} existing schemas from " - f"library '{FLOW_TOOLKIT_LIBRARY_ID}'." + f"Retrieved {len(existing_schema_types)} existing schemas for " + f"collection '{collection_id}'." ) except GQLAPIError as e: - if "NOT_FOUND" in str(e) or "do not have permission" in str(e): - logger.info( - f"Library '{FLOW_TOOLKIT_LIBRARY_ID}' not found. Assuming no existing schemas." - ) - existing_schema_types = set() - else: - raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e + raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e except (FlowConnectionError, ValidationError) as e: raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e From 6aa838c58d15718cd4e4c76dccd2fa0c333a63c5 Mon Sep 17 00:00:00 2001 From: chenm1 Date: Wed, 8 Jul 2026 15:10:09 -0700 Subject: [PATCH 3/7] Address comments --- python/tank_vendor/flow_integration_sdk/schema_builder.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/tank_vendor/flow_integration_sdk/schema_builder.py b/python/tank_vendor/flow_integration_sdk/schema_builder.py index 637393da1..0399aa891 100644 --- a/python/tank_vendor/flow_integration_sdk/schema_builder.py +++ b/python/tank_vendor/flow_integration_sdk/schema_builder.py @@ -565,11 +565,9 @@ def create_pipeline_schemas(project_id: str, config_path: str): existing_schema_types.update(schema_query.schema_types_iterator) logger.info( f"Retrieved {len(existing_schema_types)} existing schemas for " - f"collection '{collection_id}'." + f'collection "{collection_id}".' ) - except GQLAPIError as e: - raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e - except (FlowConnectionError, ValidationError) as e: + except (GQLAPIError, FlowConnectionError, ValidationError) as e: raise RuntimeError(f"Failed to retrieve existing schemas: {e}") from e # Check if schema listed in config.json already exists @@ -580,7 +578,7 @@ def create_pipeline_schemas(project_id: str, config_path: str): schema_type_id = get_schema_id(schema_dict["name"]) if schema_type_id not in existing_schema_types: logger.info( - f"Schema '{schema_type_id}' not found. Adding to list to be created." + f'Schema "{schema_type_id}" not found. Adding to list to be created.' ) need_to_create.append(schema_dict) From 44bbb90dffdf046879ffe1824b3139f518a495c9 Mon Sep 17 00:00:00 2001 From: Yungsiow Yang Date: Wed, 15 Jul 2026 14:07:11 -0400 Subject: [PATCH 4/7] SG-43935 data model derivative update (#1120) * update dcc workfile and derivative create/publish workflow - add new "variantSet" and "source" components to schema config - removed unnecessary calls to ensure_unique_name() when creating asset hierarchy - replace Task folder in asset hierarchy with a root asset which is of type "container" for now - necessary in order for the Asset Viewer to display its children - move generic assets directly under pipeline step in asset hierarchy - added FlowAsset.get_derivatives() function - switched existing derivative workflow to use new "source" component instead of previous temporary component for designating derivative source - use get_schema_id() within ComponentSpecs to avoid needing to pass in type_id parameter for components using custom schemas - added components_action parameter to publish_new_revision() so that user can choose to append components rather than replace them (which is default behaviour) - formatting for schema_builder.py * formatting * fix: correct reference-2.0.0 property serialization in component specs Properties typed as autodesk.me:reference-2.0.0 are nested schema objects, not primitives. The server expects {"objectId": {"id": ""}} rather than a plain string, matching the autodesk.data:reference-2.0.0 schema structure. - Add build_reference_value() utility to utils.py to centralize this format - Fix DerivativeSourceComponentSpec.targetVersion to use build_reference_value() - Fix VariantSetComponentSpec.targetAsset to use build_reference_value() * fix: update derivative query filter to match reference-2.0.0 schema structure The targetVersion property is typed as autodesk.me:reference-2.0.0, whose objectId field is autodesk.data:identifier-1.0.0 (a nested object with an id field), not a plain string. After fixing the serialization in DerivativeSourceComponentSpec, the RSQL filter path must reflect the new structure: data.targetVersion.objectId.id instead of data.targetVersion. Without this fix, get_derivatives() always returned empty because the old filter path never matched the nested object, causing a new derivative asset to be created on every publish instead of versioning the existing one. * Move build_reference_value from utils.py to ComponentSpec class * added component utilities - added get_sources() and get_variant_sets() utilities to ComponentMixin - handle reference properties in FlowComponent so that they're more useable. --------- Co-authored-by: chenm1 --- python/tank/flowam/config.json | 42 +++++- python/tank/flowam/create.py | 78 +++++------ .../flow_integration_sdk/globals.py | 5 +- .../flow_integration_sdk/objects.py | 124 ++++++++++++++---- .../flow_integration_sdk/publish.py | 107 +++++++++++++-- .../flow_integration_sdk/schema.py | 6 +- .../flow_integration_sdk/schema_builder.py | 20 +-- 7 files changed, 285 insertions(+), 97 deletions(-) diff --git a/python/tank/flowam/config.json b/python/tank/flowam/config.json index 16d3ff4b1..3876b3788 100644 --- a/python/tank/flowam/config.json +++ b/python/tank/flowam/config.json @@ -1,6 +1,6 @@ { "name": "demo_pipeline", - "version": "1.0.2", + "version": "1.0.3", "description": "Flow Toolkit pipeline schema definitions", "schemas": [ { @@ -183,6 +183,46 @@ "data_type": "String" } ] + }, + { + "name": "component.variantSet", + "version": "1.0.0", + "display_name": "Variant Set", + "kind": "component", + "description": "The target asset is a selectable realization of this asset for setName=variantName.", + "inherits": [], + "properties": [ + { + "name": "setName", + "data_type": "String" + }, + { + "name": "variantName", + "data_type": "String" + }, + { + "name": "targetAsset", + "data_type": "$id:autodesk.me:reference-2.0.0" + }, + { + "name": "displayName", + "data_type": "String" + } + ] + }, + { + "name": "component.source", + "version": "1.0.0", + "display_name": "Source", + "kind": "component", + "description": "The target asset represents provenance or derivation lineage.", + "inherits": [], + "properties": [ + { + "name": "targetVersion", + "data_type": "$id:autodesk.me:reference-2.0.0" + } + ] } ] } diff --git a/python/tank/flowam/create.py b/python/tank/flowam/create.py index 125989e88..f8e994d24 100644 --- a/python/tank/flowam/create.py +++ b/python/tank/flowam/create.py @@ -106,7 +106,7 @@ def get_or_create_root_folder(inputs: BaseInputs) -> FlowAsset: if not folder: logger.info(f'Creating "{SHOT_TYPE}" folder...') raw_asset = publish_new_asset( - name=ensure_unique_name(SHOT_TYPE, project), + name=SHOT_TYPE, parent_id=project.id, description="Folder for Shot assets.", components=[ @@ -119,7 +119,7 @@ def get_or_create_root_folder(inputs: BaseInputs) -> FlowAsset: if not folder: logger.info(f'Creating "{ASSET_FOLDER}" folder...') raw_asset = publish_new_asset( - name=ensure_unique_name(ASSET_FOLDER, project), + name=ASSET_FOLDER, parent_id=project.id, description="Folder for Asset Build assets.", components=[ @@ -132,7 +132,7 @@ def get_or_create_root_folder(inputs: BaseInputs) -> FlowAsset: if not folder: logger.info(f'Creating "{GENERIC_FOLDER}" folder...') raw_asset = publish_new_asset( - name=ensure_unique_name(GENERIC_FOLDER, project), + name=GENERIC_FOLDER, parent_id=project.id, description="Folder for Generic assets.", components=[ @@ -164,59 +164,59 @@ def get_or_create_workfile_parent( sg_entity_type = inputs.sg_entity_type sg_entity_name = inputs.sg_entity_name sg_pipeline_step = inputs.sg_pipeline_step - sg_task_name = inputs.sg_task_name container = root_folder.find_child(sg_entity_name) + container_type = ( + ASSET_CONTAINER_TYPE if sg_entity_type == ASSET_TYPE else SHOT_CONTAINER_TYPE + ) if not container: logger.info( f'Creating container asset for "{sg_entity_name}" under ' f'folder "{root_folder.name}"...' ) - container_type = ( - ASSET_CONTAINER_TYPE - if sg_entity_type == ASSET_TYPE - else SHOT_CONTAINER_TYPE - ) - raw_asset = publish_new_asset( - name=ensure_unique_name(sg_entity_name, root_folder), + medm_asset = publish_new_asset( + name=sg_entity_name, parent_id=root_folder.id, components=[ - TypeComponentSpec( - type_id=get_schema_id(container_type), name=f"Type {container_type}" - ) + TypeComponentSpec(type_id=get_schema_id(container_type), name=f"Type") ], ) - container = FlowAsset(raw_asset) + container = FlowAsset(medm_asset) pipeline_step = container.find_child(sg_pipeline_step) if not pipeline_step: - logger.info(f'Creating pipeline step asset for "{sg_pipeline_step}"...') - raw_asset = publish_new_asset( - name=ensure_unique_name(sg_pipeline_step, container), + logger.info(f'Creating pipeline step folder for "{sg_pipeline_step}"...') + medm_asset = publish_new_asset( + name=sg_pipeline_step, parent_id=container.id, - components=[ - TypeComponentSpec( - type_id=get_schema_id(PIPELINE_STEP_TYPE), - name=f"Type {PIPELINE_STEP_TYPE}", - ) - ], + components=[TypeComponentSpec(type_id=FOLDER_TYPE_ID, name="Type")], ) - pipeline_step = FlowAsset(raw_asset) - - task_folder = pipeline_step.find_child(sg_task_name) - if not task_folder: - logger.info(f'Creating task folder asset for "{sg_task_name}"...') - raw_asset = publish_new_asset( - name=ensure_unique_name(sg_task_name, pipeline_step), - parent_id=pipeline_step.id, - description=f'Folder for task "{sg_task_name}".', - components=[ - TypeComponentSpec(type_id=FOLDER_TYPE_ID, name=f"Type {FOLDER_TYPE_ID}") - ], - ) - task_folder = FlowAsset(raw_asset) + pipeline_step = FlowAsset(medm_asset) + + if inputs.create_mode == CreateMode.GENERIC: + # Parent generic assets directly under pipeline step + parent = pipeline_step + else: + # For dcc assets, parent them under a root asset + # that will house the dcc asset as one of its "representations" + # NOTE: the root asset will be container type for now + asset_root = pipeline_step.find_child(sg_entity_name) + if not asset_root: + logger.info(f'Creating root asset for "{sg_entity_name}"...') + medm_asset = publish_new_asset( + name=sg_entity_name, + parent_id=pipeline_step.id, + description=f'Root asset for "{sg_entity_name}".', + components=[ + TypeComponentSpec( + type_id=get_schema_id(container_type), name=f"Type" + ) + ], + ) + asset_root = FlowAsset(medm_asset) + parent = asset_root - return task_folder + return parent def ensure_unique_name(name: str, parent: FlowAsset | FlowProject) -> str: diff --git a/python/tank_vendor/flow_integration_sdk/globals.py b/python/tank_vendor/flow_integration_sdk/globals.py index 69b4d7994..8f9001e4b 100644 --- a/python/tank_vendor/flow_integration_sdk/globals.py +++ b/python/tank_vendor/flow_integration_sdk/globals.py @@ -39,9 +39,6 @@ BASE_TYPE_ID = "autodesk.me:type-1.1.0" BINARY_TYPE_ID = "autodesk.me:component.binary-1.0.0" COMMENT_TYPE_ID = "autodesk.me:component.publishComment-1.0.0" -# NOTE: This is a temporary schema being annexed for representing derivative source -# which should be switched for a dedicated schema later. -DER_SOURCE_TYPE_ID = "autodesk.me:component.dynamicPlaylistSource-1.0.0" FOLDER_TYPE_ID = "autodesk.me:type.folder-1.0.0" IMAGE_TYPE_ID = "autodesk.me:component.binary.image-1.0.0" @@ -58,6 +55,8 @@ # This should be a temporary measure, only necessary while some types # are not yet added to the autodesk domain, and must be created per collection. FILE_SEQ_TYPE = "type.fileSequence" +DER_SOURCE_TYPE = "component.source" +VARIANT_SET_TYPE = "component.variantSet" # Component purposes diff --git a/python/tank_vendor/flow_integration_sdk/objects.py b/python/tank_vendor/flow_integration_sdk/objects.py index ad27fcb23..893711d61 100644 --- a/python/tank_vendor/flow_integration_sdk/objects.py +++ b/python/tank_vendor/flow_integration_sdk/objects.py @@ -44,11 +44,14 @@ BASE_TYPE_ID, BINARY_TYPE_ID, COMMENT_TYPE_ID, - DER_SOURCE_TYPE_ID, + DER_SOURCE_COMP, + DER_SOURCE_TYPE, get_client, get_webapp_url, + VARIANT_SET_TYPE, ) from .sandbox import CheckoutDraftInfo, get_asset_drafts +from .schema import get_schema_id from .storage import ( _cache_asset_info, get_storage_asset_dir, @@ -373,6 +376,57 @@ def find_component( return matches[0] return None + @trace + def get_sources(self) -> list[str]: + """Find any Source components within the component list and + return a list of target versions that they point to. + + NOTE: A source relationship designates provenance. The target + source cab be considered the entity from which this entity + was derived. + + Returns: + List of version ids that this revision designates as a source. + + Raises: + FlowError + """ + source_type_id = get_schema_id(DER_SOURCE_TYPE) + source_comps = self.find_components(type_id=source_type_id) + try: + return [c.properties["targetVersion"] for c in source_comps] + except KeyError as exc: + msg = f"Malformed source component detected. {exc}" + raise FlowError(msg) from exc + + @trace + def get_variant_sets(self) -> dict[str, list[tuple[str, str]]]: + """Find any VariantSet components within the component list and + return a dictionary of variant sets of the structure + + set name -> list of variants + + where each variant is a tuple (variant name, asset id). + + Returns: + Dictionary of variant set names to lists of variants. + """ + varset_type_id = get_schema_id(VARIANT_SET_TYPE) + varset_comps = self.find_components(type_id=varset_type_id) + varsets = {} + try: + for comp in varset_comps: + set_name = comp.properties["setName"] + variant_name = comp.properties["variantName"] + variant_id = comp.properties["targetAsset"] + if set_name not in varsets: + varsets[set_name] = [] + varsets[set_name].append((variant_name, variant_id)) + except KeyError as exc: + msg = f"Malformed variant set component detected. {exc}" + raise FlowError(msg) from exc + return varsets + class FlowProject(FlowEntity): """Container class for data relevant to a particular medm_model.Project. @@ -820,38 +874,25 @@ def iterate_versions(self, refresh: bool = False) -> Iterator[FlowVersion]: raise FlowError(msg) from exc @trace - def find_derivative( - self, - target_type_id: str, - target_component_name: str, - ) -> FlowAsset | None: - """Search asset to find an outbound derivative where the target - matches the criteria provided. + def get_derivatives(self) -> list[FlowAsset]: + """Find siblings of this asset that were derived from it. + (i.e. have a Source component that points to this asset) - This function searches across ALL revisions of the source asset to find - any existing derivative relationship. - - Args: - target_type_id: Type of target revision. - target_component_name: Name of component on target revision to be matched. - (Derivative relationships are component to component.) - - Returns: - The first derivative asset found, or None. + Raises: + FlowError """ - # This target id should match the beginning of any revision id belonging to this asset - target_id = self.id.replace(self.MEDM_ENTITY, FlowRevision.MEDM_ENTITY) - # Generate a query to find assets which contain a source-derivative - # component with a matching target id + # This target id should match the beginning of any version id belonging to this asset + target_id = self.id.replace(self.MEDM_ENTITY, FlowVersion.MEDM_ENTITY) + der_source_type_id = get_schema_id(DER_SOURCE_TYPE) + + # Generate a query to find assets which contain a Source component with a matching target id # Since we know derivative assets will be siblings of the current asset # we can safely scope this query to the parent asset with depth of 1. client = get_client() - q_filter = f"has.component.type=={DER_SOURCE_TYPE_ID};" - q_filter += f"components[typeId:{DER_SOURCE_TYPE_ID}].data.folder.objectId=like={target_id}*;" - q_filter += ( - f"components[typeId:{DER_SOURCE_TYPE_ID}].name=='{target_component_name}'" - ) + q_filter = f"has.component.type=={der_source_type_id};" + q_filter += f"components[typeId:{der_source_type_id}].data.targetVersion.objectId.id=like={target_id}*;" + q_filter += f"components[typeId:{der_source_type_id}].name=='{DER_SOURCE_COMP}'" q_input = medm_model.AssetsByTraversalInput( start_at_id=self.parent_id, # search under parent depth=1, # search immediate children only @@ -871,6 +912,24 @@ def find_derivative( der_assets = [ FlowAsset(a) for a in q_derivatives.assets if a.id != self.parent_id ] + return der_assets + + @trace + def find_derivative( + self, + target_type_id: str, + ) -> FlowAsset | None: + """Search asset to find an outbound derivative where the target + matches the criteria provided. + + Args: + target_type_id: Type of target revision. + + Returns: + The first derivative asset found, or None. + """ + # Get list of assets that are derivatives of this asset + der_assets = self.get_derivatives() # Now filter out derivatives of the wrong type der_assets = [a for a in der_assets if target_type_id in a.type_ids] @@ -1362,7 +1421,16 @@ def __init__(self, revision: FlowRevision, component: medm_model.Component): for prop, val in component.data.items(): if prop in ["data", "purpose"]: continue # already processed these - self.properties[prop] = val + if isinstance(val, dict) and "reference" in val["typeid"]: + try: + # For reference typed properties, store the target entity id + # as the property value + self.properties[prop] = val["objectId"]["id"] + except KeyError: + # Ignore malformed properties + continue + else: + self.properties[prop] = val @trace def get_blob_path(self, blob_index: int = 0) -> str: diff --git a/python/tank_vendor/flow_integration_sdk/publish.py b/python/tank_vendor/flow_integration_sdk/publish.py index 4e8224631..9fcb49b6f 100644 --- a/python/tank_vendor/flow_integration_sdk/publish.py +++ b/python/tank_vendor/flow_integration_sdk/publish.py @@ -40,7 +40,8 @@ COMMENT_COMP, COMMENT_TYPE_ID, DER_SOURCE_COMP, - DER_SOURCE_TYPE_ID, + DER_SOURCE_TYPE, + FILE_SEQ_TYPE, get_client, IMAGE_TYPE_ID, SOURCE_COMP, @@ -48,8 +49,9 @@ THUMBNAIL_COMP, THUMBNAIL_PURPOSE, TYPE_COMP, + VARIANT_SET_TYPE, ) -from .schema import is_sub_type +from .schema import get_schema_id, is_sub_type from .storage import ( _cache_asset_info, _find_component, @@ -104,6 +106,26 @@ def create_component( """ return medm_model.ComponentDataInput(name=name, data=data, type_id=type_id) + @staticmethod + def build_reference_value(object_id: str) -> dict: + """Build a component property value for a reference-2.0.0 typed property. + + Build the nested object required for component properties typed as + autodesk.me:reference-2.0.0, which inherits from autodesk.data:reference-2.0.0 + and stores the referenced entity ID under objectId.id. + + Args: + object_id: The URN or ID of the entity being referenced. + + Returns: + Dict matching the autodesk.data:reference-2.0.0 schema structure. + + Examples: + >>> ComponentSpec.build_reference_value("urn:medm:asset:c:p:abc") + {'objectId': {'id': 'urn:medm:asset:c:p:abc'}} + """ + return {"objectId": {"id": object_id}} + class BinaryComponentSpec(ComponentSpec): """Base class for all binary component specs.""" @@ -249,12 +271,12 @@ class DerivativeSourceComponentSpec(ComponentSpec): There is only expected to be one of these per revision. """ - def __init__(self, revision_id: str): + def __init__(self, version_id: str): """ Args: - revision_id: Id of source revision. + version_id: Id of source version. """ - self.revision_id = revision_id + self.version_id = version_id @property def name(self) -> str: @@ -264,8 +286,8 @@ def create(self) -> medm_model.Component: """Create an MEDM component based on specifications.""" return self.create_component( name=self.name, - type_id=DER_SOURCE_TYPE_ID, - folder={"objectId": self.revision_id}, + type_id=get_schema_id(DER_SOURCE_TYPE), + targetVersion=self.build_reference_value(self.version_id), ) @@ -363,6 +385,64 @@ def create(self) -> medm_model.Component: ) +class VariantSetComponentSpec(ComponentSpec): + """Specifications for creating a variant set component. + This is a component used in a conceptually atomic asset + to list available variant sets and variants for the asset. + There can be multiple such components on an asset with + that may span multiple variant sets. + + Example: If an asset were to have two orthogonal variant + sets for model representation, and surfacing look + as outlined below. + + -> Set 1 = Representation + - Variant (Maya) + - Variant (Alembic) + -> Set 2 = Look + - Variant (Default) + - Variant (Dirty) + + This would require four VariantSet components to + be added to the asset. + + -> VariantSet 1 = Representation-Maya + -> VariantSet 2 = Representation-Alembic + -> VariantSet 3 = Look-Default + -> VariantSet 4 = Look-Dirty + """ + + def __init__( + self, set_name: str, variant_name: str, asset_id: str, display_name: str = "" + ): + """ + Args: + set_name: Name of variant set under which this variant belongs. + variant_name: Name of this specific variant under the set name. + asset_id: The asset which represents this variant. + display_name: An optional display name for this set+variant combo. + """ + self.set_name = set_name + self.variant_name = variant_name + self.asset_id = asset_id + self.display_name = display_name + + @property + def name(self) -> str: + return f"{self.set_name}-{self.variant_name}" + + def create(self) -> medm_model.Component: + """Create an MEDM component based on specifications.""" + return self.create_component( + name=self.name, + type_id=get_schema_id(VARIANT_SET_TYPE), + setName=self.set_name, + variantName=self.variant_name, + targetAsset=self.build_reference_value(self.asset_id), + displayName=self.display_name, + ) + + class FileSeqComponentSpec(TypeComponentSpec): """Specifications for creating a file sequence type component. This is a component used to designate an asset as containing a file sequence. @@ -371,7 +451,6 @@ class FileSeqComponentSpec(TypeComponentSpec): def __init__( self, - type_id: str, frame_start: int, frame_end: int, frame_set: str, @@ -380,7 +459,6 @@ def __init__( ): """ Args: - type_id: The MEDM type identifier for the type component. frame_start: First frame of file sequence. frame_end: End frame of file sequence. frame_set: A string expression denoting the set of frames @@ -401,11 +479,11 @@ def __init__( # global constant and use that instead. # If provided, type id must be subtype of base type + type_id = get_schema_id(FILE_SEQ_TYPE) if not is_sub_type(BASE_TYPE_ID, type_id): msg = f"Type id {type_id} is not a subclass of base type." raise ComponentSpecError(details=msg) - self.type_id = type_id self.frame_start = frame_start self.frame_end = frame_end self.frame_set = frame_set @@ -421,7 +499,7 @@ def create(self) -> medm_model.Component: """Create an MEDM component based on specifications.""" return self.create_component( name=self.name, - type_id=self.type_id, + type_id=get_schema_id(FILE_SEQ_TYPE), frameStart=self.frame_start, frameEnd=self.frame_end, frameSet=self.frame_set, @@ -496,6 +574,7 @@ def publish_new_revision( asset_id: str, components: list[ComponentSpec] | None = None, used_versions: list[str] | None = None, + components_action: medm_model.ListAction = medm_model.ListAction.REPLACE, ) -> medm_model.Asset: """Publish a new version of an existing asset. @@ -506,6 +585,10 @@ def publish_new_revision( (Components are used to store binaries and metadata on revisions.) used_versions: List of version ids of other assets used by this asset. (Stored as "uses" relationships with other asset versions.) + components_action: Should component list replace existing components or append to them? + Valid values are: + * ListAction.REPLACE (default) + * ListAction.ADD Returns: Updated asset object. @@ -524,7 +607,7 @@ def publish_new_revision( m_input = medm_model.UpdateAssetInput( id=asset_id, components=medm_components, - components_action=medm_model.ListAction.REPLACE.value, + components_action=components_action.value, named_version_change=medm_model.NamedVersionChangeEnum.CREATE_NEW.value, uses=medm_uses, ) diff --git a/python/tank_vendor/flow_integration_sdk/schema.py b/python/tank_vendor/flow_integration_sdk/schema.py index 4b6188966..4a999181c 100644 --- a/python/tank_vendor/flow_integration_sdk/schema.py +++ b/python/tank_vendor/flow_integration_sdk/schema.py @@ -28,7 +28,6 @@ BASE_TYPE_ID, BINARY_TYPE_ID, COMMENT_TYPE_ID, - DER_SOURCE_TYPE_ID, FOLDER_TYPE_ID, get_client, get_session_collection, @@ -51,7 +50,6 @@ _schema_tree[BASE_TYPE_ID] = [] _schema_tree[BINARY_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] _schema_tree[COMMENT_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] -_schema_tree[DER_SOURCE_TYPE_ID] = [BASE_COMPONENT_TYPE_ID] _schema_tree[FOLDER_TYPE_ID] = [BASE_TYPE_ID] _schema_tree[IMAGE_TYPE_ID] = [BINARY_TYPE_ID] @@ -128,9 +126,7 @@ def cache_schema_config(config_path: str): type_name = schema.get("name", "") kind = schema.get("kind") if not kind: - raise ValueError( - f"Schema '{type_name}' is missing required 'kind' field." - ) + raise ValueError(f"Schema '{type_name}' is missing required 'kind' field.") parent_types = schema.get("inherits", []) # strip "$ref:" prefix parent_types = [pt[5:] for pt in parent_types] diff --git a/python/tank_vendor/flow_integration_sdk/schema_builder.py b/python/tank_vendor/flow_integration_sdk/schema_builder.py index 0399aa891..6af8f2414 100644 --- a/python/tank_vendor/flow_integration_sdk/schema_builder.py +++ b/python/tank_vendor/flow_integration_sdk/schema_builder.py @@ -274,9 +274,7 @@ def build(self) -> flow_model.Schema: # Build property list for CreateSchemaInput properties = [] for property_dict in self.schema_dict.get("properties", []): - properties.append( - self._create_properties_definition_input(property_dict) - ) + properties.append(self._create_properties_definition_input(property_dict)) try: # Create CreateSchemaInput for the mutation @@ -323,7 +321,9 @@ def get_display_data(self) -> flow_model.SchemaDisplayData | None: """ if not self.schema: - raise FlowSchemaBuilderError(details="Schema instance has not been built yet.") + raise FlowSchemaBuilderError( + details="Schema instance has not been built yet." + ) client = get_client() schema_display_data = None @@ -372,7 +372,9 @@ def update_display_name(self) -> str | None: display_name = self.schema_dict.get("display_name") if not self.schema: - raise FlowSchemaBuilderError(details="Schema instance has not been built yet.") + raise FlowSchemaBuilderError( + details="Schema instance has not been built yet." + ) if not display_name: raise FlowSchemaBuilderError( @@ -494,9 +496,7 @@ def _create_schema_library( project_id=project_id, ) # Create and call the schema mutation - schema_query = client.service_schema.create_schema_library( - variables=input_data - ) + schema_query = client.service_schema.create_schema_library(variables=input_data) query_response = schema_query.call() # Extract the created schema library from the response schema_library = query_response.schema_library @@ -542,7 +542,9 @@ def create_pipeline_schemas(project_id: str, config_path: str): config = schema._read_schema_config(config_path) if "schemas" not in config: - raise KeyError("The schema config file must contain a 'schemas' key with a list of schemas to create.") + raise KeyError( + "The schema config file must contain a 'schemas' key with a list of schemas to create." + ) client = get_client() collection_id = FlowProject.get_collection_id(project_id) From ae7c303524ff37b226ada259183fa21b848666ec Mon Sep 17 00:00:00 2001 From: Ming Chen Date: Thu, 16 Jul 2026 11:57:46 -0700 Subject: [PATCH 5/7] feat: [SG-43934] Introduce Layer component for asset(pipeline step) hierarchy (#1121) * feat: [SG-43934] Introduce Layer component for asset(pipeline step) hierarchy Part of the new Flow AM data model implementation. The pipeline step hierarchy no longer uses a task-folder level between the pipeline step and the workfile. Instead, a LayerComponent on the root asset container tracks each pipeline step as a named compositional relationship. Changes: - config.json: add component.layer schema (v1.0.3) with name (String) and targetAsset (reference-2.0.0) properties - globals.py: add LAYER_TYPE and LAYER_COMP constants - publish.py: add ComponentSpec.build_reference_value() static method for building reference-2.0.0 property values; add LayerComponentSpec; add components_action parameter to publish_new_revision() to support ListAction.ADD when appending layer components without replacing existing ones - objects.py: add get_layers() to FlowAsset and FlowRevision to read LayerComponents and return referenced layer folder assets - create.py: update get_or_create_workfile_parent() to create pipeline steps as FOLDER_TYPE_ID, publish a LayerComponent onto the root asset container with ListAction.ADD, and remove the task-folder level --- python/tank/flowam/config.json | 18 ++++++++++ python/tank/flowam/create.py | 21 ++++++++++-- .../flow_integration_sdk/globals.py | 1 + .../flow_integration_sdk/objects.py | 22 +++++++++++++ .../flow_integration_sdk/publish.py | 33 +++++++++++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/python/tank/flowam/config.json b/python/tank/flowam/config.json index 3876b3788..6d49f51f0 100644 --- a/python/tank/flowam/config.json +++ b/python/tank/flowam/config.json @@ -184,6 +184,24 @@ } ] }, + { + "name": "component.layer", + "version": "1.0.0", + "display_name": "Layer", + "kind": "component", + "description": "A named compositional relationship. The target asset contributes compositionally to this asset.", + "inherits": [], + "properties": [ + { + "name": "name", + "data_type": "String" + }, + { + "name": "targetAsset", + "data_type": "$id:autodesk.me:reference-2.0.0" + } + ] + }, { "name": "component.variantSet", "version": "1.0.0", diff --git a/python/tank/flowam/create.py b/python/tank/flowam/create.py index f8e994d24..475b15ce8 100644 --- a/python/tank/flowam/create.py +++ b/python/tank/flowam/create.py @@ -13,12 +13,15 @@ import re from enum import Enum +from tank_vendor.flow_data_sdk.base import model as medm_model from tank_vendor.flow_integration_sdk.exceptions import CreateAssetError, FlowError from tank_vendor.flow_integration_sdk.globals import FOLDER_TYPE_ID from tank_vendor.flow_integration_sdk.objects import FlowAsset, FlowProject from tank_vendor.flow_integration_sdk.publish import ( + LayerComponentSpec, TypeComponentSpec, publish_new_asset, + publish_new_revision, ) from tank_vendor.flow_integration_sdk.schema import get_schema_id from tank_vendor.flow_integration_sdk.utils import get_logger, trace @@ -153,11 +156,10 @@ def get_or_create_root_folder(inputs: BaseInputs) -> FlowAsset: def get_or_create_workfile_parent( root_folder: FlowAsset, inputs: BaseInputs ) -> FlowAsset: - """Determine (and create if necessary) the task-level folder that will be - the direct parent of the workfile asset. + """Determine (and create if necessary) the parent container of the workfile asset. Returns: - The task-folder :class:`FlowAsset`. + The parent asset as :class:`FlowAsset`. """ logger = get_logger(__name__) @@ -193,6 +195,19 @@ def get_or_create_workfile_parent( ) pipeline_step = FlowAsset(medm_asset) + logger.info( + f'Adding layer component for "{sg_pipeline_step}" on ' + f'container "{container.name}"...' + ) + publish_new_revision( + asset_id=container.id, + components=[ + LayerComponentSpec( + layer_name=sg_pipeline_step, asset_id=pipeline_step.id + ) + ], + components_action=medm_model.ListAction.ADD, + ) if inputs.create_mode == CreateMode.GENERIC: # Parent generic assets directly under pipeline step parent = pipeline_step diff --git a/python/tank_vendor/flow_integration_sdk/globals.py b/python/tank_vendor/flow_integration_sdk/globals.py index 8f9001e4b..79ed095f1 100644 --- a/python/tank_vendor/flow_integration_sdk/globals.py +++ b/python/tank_vendor/flow_integration_sdk/globals.py @@ -56,6 +56,7 @@ # are not yet added to the autodesk domain, and must be created per collection. FILE_SEQ_TYPE = "type.fileSequence" DER_SOURCE_TYPE = "component.source" +LAYER_TYPE = "component.layer" VARIANT_SET_TYPE = "component.variantSet" diff --git a/python/tank_vendor/flow_integration_sdk/objects.py b/python/tank_vendor/flow_integration_sdk/objects.py index 893711d61..abd52b0c6 100644 --- a/python/tank_vendor/flow_integration_sdk/objects.py +++ b/python/tank_vendor/flow_integration_sdk/objects.py @@ -48,6 +48,7 @@ DER_SOURCE_TYPE, get_client, get_webapp_url, + LAYER_TYPE, VARIANT_SET_TYPE, ) from .sandbox import CheckoutDraftInfo, get_asset_drafts @@ -427,6 +428,27 @@ def get_variant_sets(self) -> dict[str, list[tuple[str, str]]]: raise FlowError(msg) from exc return varsets + @trace + def get_layers(self) -> dict[str, str]: + """Find any Layer components and return a dictionary of layer names + mapped to the asset id of each layer asset. + + Returns: + Dictionary mapping layer name to asset id. + """ + layer_type_id = get_schema_id(LAYER_TYPE) + layer_comps = self.find_components(type_id=layer_type_id) + layers = {} + try: + for comp in layer_comps: + layer_name = comp.properties["layerName"] + layer_id = comp.properties["targetAsset"] + layers[layer_name] = layer_id + except KeyError as exc: + msg = f"Malformed layer component detected. {exc}" + raise FlowError(msg) from exc + return layers + class FlowProject(FlowEntity): """Container class for data relevant to a particular medm_model.Project. diff --git a/python/tank_vendor/flow_integration_sdk/publish.py b/python/tank_vendor/flow_integration_sdk/publish.py index 9fcb49b6f..a85284b6d 100644 --- a/python/tank_vendor/flow_integration_sdk/publish.py +++ b/python/tank_vendor/flow_integration_sdk/publish.py @@ -44,6 +44,7 @@ FILE_SEQ_TYPE, get_client, IMAGE_TYPE_ID, + LAYER_TYPE, SOURCE_COMP, SOURCE_PURPOSE, THUMBNAIL_COMP, @@ -507,6 +508,38 @@ def create(self) -> medm_model.Component: ) +class LayerComponentSpec(ComponentSpec): + """Specifications for creating a layer component. + + A layer component represents a named compositional relationship where the + target asset contributes compositionally to this asset. It carries a + ``targetAsset`` reference to the contributing child asset. + """ + + def __init__(self, layer_name: str, asset_id: str): + """ + Args: + layer_name: Name identifying this layer relationship + (e.g. the pipeline-step name). + asset_id: MEDM id of the target asset the layer points to. + """ + self.layer_name = layer_name + self.asset_id = asset_id + + @property + def name(self) -> str: + return f"Layer-{self.layer_name}" + + def create(self) -> medm_model.Component: + """Create an MEDM component based on specifications.""" + return self.create_component( + name=self.name, + type_id=get_schema_id(LAYER_TYPE), + layerName=self.layer_name, + targetAsset=self.build_reference_value(self.asset_id), + ) + + @trace def publish_new_asset( name: str, From f3159c8df654504e76ed42acf12c5786aa97612c Mon Sep 17 00:00:00 2001 From: Carlos Villavicencio Date: Tue, 21 Jul 2026 11:57:22 -0500 Subject: [PATCH 6/7] SG-43936 [Data Model] Add Reference components on publish (#1115) * [Data Model] Add Reference components on publish * update dcc workfile and derivative create/publish workflow - add new "variantSet" and "source" components to schema config - removed unnecessary calls to ensure_unique_name() when creating asset hierarchy - replace Task folder in asset hierarchy with a root asset which is of type "container" for now - necessary in order for the Asset Viewer to display its children - move generic assets directly under pipeline step in asset hierarchy - added FlowAsset.get_derivatives() function - switched existing derivative workflow to use new "source" component instead of previous temporary component for designating derivative source - use get_schema_id() within ComponentSpecs to avoid needing to pass in type_id parameter for components using custom schemas - added components_action parameter to publish_new_revision() so that user can choose to append components rather than replace them (which is default behaviour) - formatting for schema_builder.py * formatting * fix: correct reference-2.0.0 property serialization in component specs Properties typed as autodesk.me:reference-2.0.0 are nested schema objects, not primitives. The server expects {"objectId": {"id": ""}} rather than a plain string, matching the autodesk.data:reference-2.0.0 schema structure. - Add build_reference_value() utility to utils.py to centralize this format - Fix DerivativeSourceComponentSpec.targetVersion to use build_reference_value() - Fix VariantSetComponentSpec.targetAsset to use build_reference_value() * fix: update derivative query filter to match reference-2.0.0 schema structure The targetVersion property is typed as autodesk.me:reference-2.0.0, whose objectId field is autodesk.data:identifier-1.0.0 (a nested object with an id field), not a plain string. After fixing the serialization in DerivativeSourceComponentSpec, the RSQL filter path must reflect the new structure: data.targetVersion.objectId.id instead of data.targetVersion. Without this fix, get_derivatives() always returned empty because the old filter path never matched the nested object, causing a new derivative asset to be created on every publish instead of versioning the existing one. * Move build_reference_value from utils.py to ComponentSpec class * added component utilities - added get_sources() and get_variant_sets() utilities to ComponentMixin - handle reference properties in FlowComponent so that they're more useable. * Fix import * Code review feedback * Code review feedback * More feedback * Refactor `get_references` * Update message --------- Co-authored-by: Yungsiow Yang Co-authored-by: chenm1 --- python/tank/flowam/config.json | 14 +++++++++ python/tank/flowam/utils.py | 30 ++++++++++++++++++ .../flow_integration_sdk/globals.py | 1 + .../flow_integration_sdk/objects.py | 23 ++++++++++++++ .../flow_integration_sdk/publish.py | 31 +++++++++++++++++++ 5 files changed, 99 insertions(+) diff --git a/python/tank/flowam/config.json b/python/tank/flowam/config.json index 6d49f51f0..b719a805c 100644 --- a/python/tank/flowam/config.json +++ b/python/tank/flowam/config.json @@ -241,6 +241,20 @@ "data_type": "$id:autodesk.me:reference-2.0.0" } ] + }, + { + "name": "component.reference", + "version": "1.0.0", + "display_name": "Reference", + "kind": "component", + "description": "The target asset is referenced as a dependency or included asset.", + "inherits": [], + "properties": [ + { + "name": "targetVersion", + "data_type": "$id:autodesk.me:reference-2.0.0" + } + ] } ] } diff --git a/python/tank/flowam/utils.py b/python/tank/flowam/utils.py index a0d56c625..2f323c316 100644 --- a/python/tank/flowam/utils.py +++ b/python/tank/flowam/utils.py @@ -32,11 +32,13 @@ from tank_vendor.flow_integration_sdk.publish import ( ComponentSpec, CommentComponentSpec, + ReferenceComponentSpec, SourceComponentSpec, ThumbnailComponentSpec, TypeComponentSpec, FileSeqComponentSpec, ) +from tank_vendor.flow_integration_sdk.dependency import DependencyData, DepType from tank_vendor.flow_integration_sdk.objects import FlowProject from tank_vendor.flow_integration_sdk.schema_builder import create_pipeline_schemas from tank_vendor.flow_integration_sdk.utils import trace @@ -216,6 +218,7 @@ def create_components_for_publish( thumbnail_path: str = "", comment: str = "", type_ids: list[str] | None = None, + deps: list[DependencyData] | None = None, ) -> list[ComponentSpec]: """Generate the components relevant to publish a new revision. @@ -228,6 +231,11 @@ def create_components_for_publish( type_ids: A list of type ids to be converted into type components. This is only relevant if publishing a new asset direct to remote (i.e. not going through sandbox). + deps: Optional list of internal dependencies found in the scene. + These will be recorded as ReferenceComponentSpec entries on the revision. + + Raises: + FlowError """ # Source component contains the source file components: list[ComponentSpec] = [] @@ -272,6 +280,28 @@ def create_components_for_publish( # NOTE: component names must be unique! type_comp = TypeComponentSpec(type_id=type_id, name=f"Type {i}") components.append(type_comp) + # Add reference components for each dependency. + # Callers are expected to pass only resolved asset dependencies + # (dep_type == ASSET with a valid version_id). + if deps: + for i, dep in enumerate(deps): + if dep.dep_type != DepType.ASSET: + raise FlowError( + f"Dependency at index {i} has unexpected type " + f"{dep.dep_type}. Only DepType.ASSET deps should be " + "passed as reference components." + ) + if not dep.version_id: + raise FlowError( + f"Dependency at index {i} is missing a version_id. " + "Cannot create a reference component without a target version." + ) + components.append( + ReferenceComponentSpec( + name=f"Reference {i}", + version_id=dep.version_id, + ) + ) return components diff --git a/python/tank_vendor/flow_integration_sdk/globals.py b/python/tank_vendor/flow_integration_sdk/globals.py index 79ed095f1..90a27e9f1 100644 --- a/python/tank_vendor/flow_integration_sdk/globals.py +++ b/python/tank_vendor/flow_integration_sdk/globals.py @@ -55,6 +55,7 @@ # This should be a temporary measure, only necessary while some types # are not yet added to the autodesk domain, and must be created per collection. FILE_SEQ_TYPE = "type.fileSequence" +REFERENCE_TYPE = "component.reference" DER_SOURCE_TYPE = "component.source" LAYER_TYPE = "component.layer" VARIANT_SET_TYPE = "component.variantSet" diff --git a/python/tank_vendor/flow_integration_sdk/objects.py b/python/tank_vendor/flow_integration_sdk/objects.py index abd52b0c6..2157fa630 100644 --- a/python/tank_vendor/flow_integration_sdk/objects.py +++ b/python/tank_vendor/flow_integration_sdk/objects.py @@ -46,6 +46,7 @@ COMMENT_TYPE_ID, DER_SOURCE_COMP, DER_SOURCE_TYPE, + REFERENCE_TYPE, get_client, get_webapp_url, LAYER_TYPE, @@ -428,6 +429,28 @@ def get_variant_sets(self) -> dict[str, list[tuple[str, str]]]: raise FlowError(msg) from exc return varsets + @trace + def get_references(self) -> list[str]: + """Find any Reference components within the component list and + return a list of target versions that they point to. + + NOTE: A reference relationship designates a dependency on another + asset version at the time of publish. + + Returns: + List of version ids that this revision references as dependencies. + + Raises: + FlowError + """ + ref_type_id = get_schema_id(REFERENCE_TYPE) + ref_comps = self.find_components(type_id=ref_type_id) + try: + return [c.properties["targetVersion"] for c in ref_comps] + except KeyError as exc: + msg = f"Malformed reference component detected. {exc}" + raise FlowError(msg) from exc + @trace def get_layers(self) -> dict[str, str]: """Find any Layer components and return a dictionary of layer names diff --git a/python/tank_vendor/flow_integration_sdk/publish.py b/python/tank_vendor/flow_integration_sdk/publish.py index a85284b6d..0389bd2e1 100644 --- a/python/tank_vendor/flow_integration_sdk/publish.py +++ b/python/tank_vendor/flow_integration_sdk/publish.py @@ -44,6 +44,7 @@ FILE_SEQ_TYPE, get_client, IMAGE_TYPE_ID, + REFERENCE_TYPE, LAYER_TYPE, SOURCE_COMP, SOURCE_PURPOSE, @@ -292,6 +293,36 @@ def create(self) -> medm_model.Component: ) +class ReferenceComponentSpec(ComponentSpec): + """Specifications for creating a reference component. + This is a component used to record a dependency on another asset/version. + Multiple reference components may be added to a single revision, one per + dependency. + """ + + def __init__(self, name: str, version_id: str): + """ + Args: + name: Unique name for this component within the revision + (e.g. "Reference 0", "Reference 1"). + version_id: The MEDM version id of the dependency being referenced. + """ + self._name = name + self.version_id = version_id + + @property + def name(self) -> str: + return self._name + + def create(self) -> medm_model.Component: + """Create an MEDM component based on specifications.""" + return self.create_component( + name=self.name, + type_id=get_schema_id(REFERENCE_TYPE), + targetVersion=self.build_reference_value(self.version_id), + ) + + class SourceComponentSpec(BinaryComponentSpec): """Specifications for creating a source component. This is a component used to store the main source file(s) of the revision. From 5dd8df808f90ca701ebaee97b202eabaaa7542bd Mon Sep 17 00:00:00 2001 From: Ming Chen Date: Wed, 22 Jul 2026 09:09:42 -0700 Subject: [PATCH 7/7] Fix[SG-43934]: Add displayname for layer component (#1124) --- python/tank/flowam/create.py | 4 +++- python/tank_vendor/flow_integration_sdk/publish.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/tank/flowam/create.py b/python/tank/flowam/create.py index 475b15ce8..0e6f8a7d3 100644 --- a/python/tank/flowam/create.py +++ b/python/tank/flowam/create.py @@ -203,7 +203,9 @@ def get_or_create_workfile_parent( asset_id=container.id, components=[ LayerComponentSpec( - layer_name=sg_pipeline_step, asset_id=pipeline_step.id + layer_name=sg_pipeline_step, + asset_id=pipeline_step.id, + display_name=sg_pipeline_step, ) ], components_action=medm_model.ListAction.ADD, diff --git a/python/tank_vendor/flow_integration_sdk/publish.py b/python/tank_vendor/flow_integration_sdk/publish.py index 0389bd2e1..77f93d974 100644 --- a/python/tank_vendor/flow_integration_sdk/publish.py +++ b/python/tank_vendor/flow_integration_sdk/publish.py @@ -547,15 +547,17 @@ class LayerComponentSpec(ComponentSpec): ``targetAsset`` reference to the contributing child asset. """ - def __init__(self, layer_name: str, asset_id: str): + def __init__(self, layer_name: str, asset_id: str, display_name: str = ""): """ Args: layer_name: Name identifying this layer relationship (e.g. the pipeline-step name). asset_id: MEDM id of the target asset the layer points to. + display_name: An optional human-readable display name for this layer. """ self.layer_name = layer_name self.asset_id = asset_id + self.display_name = display_name @property def name(self) -> str: @@ -568,6 +570,7 @@ def create(self) -> medm_model.Component: type_id=get_schema_id(LAYER_TYPE), layerName=self.layer_name, targetAsset=self.build_reference_value(self.asset_id), + displayName=self.display_name, )