IECoreUSD : Add option to write OSL shaders conformant with RenderMan

This means writing the `info:id` attribute without our `osl:` prefix, and omitting any directories in the shader name. Since this is a breaking change, it is gated behind an IECOREUSD_WRITE_CONFORMANT_OSL_SHADERS environment variable for folks to opt in via.

Unfortunately Arnold doesn't support shaders exported in this format, but it didn't support the old format either. Since the new format seems to be "blessed" by Pixar, perhaps we can contribute support for it to `arnold-usd` in the future. Alternatively, we could write OSL shaders as `arnold:osl` shaders, but only when assigned via an `arnold:` prefixed material binding. This is left as future work though.

Adopting the RenderMan-style formatting for OSL shaders implies a few things for our own small shader library in Gaffer :

- We should move to a flat directory structure, so we don't need to save the full name in the `cortex:shaderName` sidecar metadata.
- We should prefix the shader names with "Gaffer", since the likelihood of collision is much greater without the directory names.

This is also left as future work though.

Author: johnhaddon

Changes

@@ -1,7 +1,10 @@
 10.6.x.x (relative to 10.6.5.0)
 ========
 
+Improvements
+------------
 
+- USDScene : Added `IECOREUSD_WRITE_CONFORMANT_OSL_SHADERS` environment variable. If set to a value of `1`, this causes OSL shaders to be written in the format expected by RenderMan. Set the `RMAN_SHADERPATH` environment variable appropriately to ensure RenderMan can find the shaders.
 
 10.6.5.0 (relative to 10.6.4.0)
 ========

contrib/IECoreUSD/src/IECoreUSD/ShaderAlgo.cpp

@@ -56,6 +56,7 @@
 #include "boost/algorithm/string/replace.hpp"
 #include "boost/pointer_cast.hpp"
 
+#include <filesystem>
 #include <regex>
 
 #if PXR_VERSION < 2102
@@ -66,26 +67,68 @@ namespace
 {
 
 const pxr::TfToken g_blindDataToken( "cortex:blindData" );
+const pxr::TfToken g_shaderNameToken( "cortex:shaderName" );
+const pxr::TfToken g_shaderTypeToken( "cortex:shaderType" );
 pxr::TfToken g_legacyAdapterLabelToken( IECoreScene::ShaderNetworkAlgo::componentConnectionAdapterLabel().string() );
 
-std::pair<pxr::TfToken, std::string> shaderIdAndType( const pxr::UsdShadeConnectableAPI &connectable )
+bool writeConformantOSLShaders()
+{
+	if( const char *e = getenv( "IECOREUSD_WRITE_CONFORMANT_OSL_SHADERS" ) )
+	{
+		return strcmp( e, "0" );
+	}
+	return false;
+}
+
+// Recover a Cortex-style shader name and type, essentially reversing the
+// transformation done by `createShaderPrim()`.
+std::pair<std::string, std::string> shaderNameAndType( const pxr::UsdShadeConnectableAPI &connectable )
 {
-	pxr::TfToken id;
-	std::string type;
 	if( auto shader = pxr::UsdShadeShader( connectable ) )
 	{
-		shader.GetShaderId( &id );
-		type = "surface";
+		std::string name;
+		std::string type;
+
+		pxr::VtValue nameVtValue = shader.GetPrim().GetCustomDataByKey( g_shaderNameToken );
+		if( !nameVtValue.IsEmpty()  )
+		{
+			name = nameVtValue.Get<std::string>();
+		}
+		else
+		{
+			pxr::TfToken id;
+			shader.GetShaderId( &id );
+			name = id.GetString();
+			type = "surface";
+			const size_t colonPos = name.find( ":" );
+			if( colonPos != std::string::npos )
+			{
+				std::string prefix = name.substr( 0, colonPos );
+				name = name.substr( colonPos + 1 );
+				if( prefix == "arnold" )
+				{
+					prefix = "ai";
+				}
+				type = prefix + ":shader";
+			}
+		}
+
+		pxr::VtValue typeVtValue = shader.GetPrim().GetCustomDataByKey( g_shaderTypeToken );
+		if( !typeVtValue.IsEmpty() )
+		{
+			type = typeVtValue.Get<std::string>();
+		}
+
+		return { name, type };
 	}
 #if PXR_VERSION >= 2111
 	else if( auto light = pxr::UsdLuxLightAPI( connectable ) )
 	{
-		light.GetShaderIdAttr().Get( &id );
-		type = "light";
+		return { light.GetShaderId( {} ).GetString(), "light" };
 	}
 #endif
 
-	return std::make_pair( id, type );
+	return { "", "" };
 }
 
 bool writeNonStandardLightParameter( const std::string &name, const IECore::Data *value, pxr::UsdShadeConnectableAPI usdShader )
@@ -232,24 +275,7 @@ IECore::InternedString readShaderNetworkWalk( const pxr::SdfPath &anchorPath, co
 		return handle;
 	}
 
-	auto [id, shaderType] = shaderIdAndType( usdShader );
-	std::string shaderName = "defaultsurface";
-	if( id.size() )
-	{
-		std::string name = id.GetString();
-		size_t colonPos = name.find( ":" );
-		if( colonPos != std::string::npos )
-		{
-			std::string prefix = name.substr( 0, colonPos );
-			name = name.substr( colonPos + 1 );
-			if( prefix == "arnold" )
-			{
-				prefix = "ai";
-			}
-			shaderType = prefix + ":shader";
-		}
-		shaderName = name;
-	}
+	// Read parameter values and connections.
 
 	IECore::CompoundDataPtr parametersData = new IECore::CompoundData();
 	IECore::CompoundDataMap &parameters = parametersData->writable();
@@ -295,6 +321,9 @@ IECore::InternedString readShaderNetworkWalk( const pxr::SdfPath &anchorPath, co
 
 	readNonStandardLightParameters( usdShader.GetPrim(), parameters );
 
+	// Create shader.
+
+	auto [shaderName, shaderType] = shaderNameAndType( usdShader );
 	IECoreScene::ShaderPtr newShader = new IECoreScene::Shader( shaderName, shaderType, parametersData );
 
 	// General purpose support for any Cortex blind data.
@@ -395,35 +424,76 @@ pxr::UsdShadeConnectableAPI createShaderPrim( const IECoreScene::Shader *shader,
 		throw IECore::Exception( "Could not create shader at " + path.GetAsString() );
 	}
 
-	const std::string type = shader->getType();
+	// We need to declare the shader in a form corresponding to entries in USD's
+	// Sdr registry, so that if rendered in `usdview` or another Hydra-based
+	// app, the render delegates can find the shaders. We could potentially do
+	// that done by finding a shader in the Sdr registry by name and then using
+	// `SdrShaderNode.GetIdentifier()` or
+	// `SdrShaderNode.GetResolvedImplementationURI()`, although it's not clear
+	// how you'd decide which to use. Anyway, for now at least, we want to be
+	// able to operate in environments without renderer-specific USD plugins
+	// installed. So instead of querying the registry we use some heuristics of
+	// our own.
 
-	std::string typePrefix;
 	if( boost::starts_with( shader->getName(), "Pxr" ) || boost::starts_with( shader->getName(), "Lama" ) )
 	{
-		// Leave the type prefix empty. This should be the default, but we are currently only doing this
-		// for a small number of shaders that we can be completely confident require it, in order to
-		// preserve backwards compatibility.
+		// Could be either an OSL or a C++ shader, but either way, RenderMan
+		// registers it in the SdrRegistry by name.
+		usdShader.SetShaderId( pxr::TfToken( shader->getName() ) );
+	}
+	else if( boost::starts_with( shader->getType(), "ai:" ) )
+	{
+		// Arnold registers all plugins at startup, so also only needs the name
+		// to be able to create a shader. It uses an `arnold:` prefix to avoid
+		// clashes with the names of other shaders.
+		usdShader.SetShaderId( pxr::TfToken( "arnold:" + shader->getName() ) );
+	}
+	else if(
+		boost::starts_with( shader->getType(), "osl:" ) &&
+		writeConformantOSLShaders()
+	)
+	{
+		// Arbitrary OSL shader. Arnold would want that written as an
+		// `arnold:osl` shader with `input:shadername` pointing to the shader.
+		// But that will never work in another renderer. RenderMan takes a
+		// slightly more generic approach by registering each OSL shader from
+		// `RMAN_SHADERPATH` into the Sdr registry, so we follow that in the hope
+		// that Arnold and other renderers might fall in line in future.
+		if( shader->getName().find( '/' ) != std::string::npos )
+		{
+			// Unfortunately, RenderMan's SdrDiscoveryPlugin uses only the leaf
+			// name, even though the Riley API will accept `{directory}/{file}`.
+			// So we write the leaf name, but also record the full name so we
+			// can round-trip back to Cortex exactly.
+			usdShader.SetShaderId( pxr::TfToken( std::filesystem::path( shader->getName() ).stem() ) );
+			usdShader.GetPrim().SetCustomDataByKey( g_shaderNameToken, pxr::VtValue( shader->getName() ) );
+		}
+		else
+		{
+			usdShader.SetShaderId( pxr::TfToken( shader->getName() ) );
+		}
+		// We've lost the fact that it was an OSL shader, so store that information
+		// as custom data for use when loading.
+		usdShader.GetPrim().SetCustomDataByKey( g_shaderTypeToken, pxr::VtValue( shader->getType() ) );
+		/// \todo We don't want to be resorting to storing custom data for the name or
+		/// the type. Rejig things so that we only use leaf names for shaders, and don't
+		/// rely on the type anywhere, and then remove the custom data.
 	}
 	else
 	{
-		size_t typeColonPos = type.find( ":" );
+		const std::string &type = shader->getType();
+		const size_t typeColonPos = type.find( ":" );
 		if( typeColonPos != std::string::npos )
 		{
-			// According to our current understanding, this is almost completely wrong. Renderer's like
-			// PRMan won't accept shaders with type prefixes, and Arnold apparently requires all shaders
-			// to be prefixed with "arnold:", including OSL. This code prefixes OSL shaders with "osl:",
-			// which fails in all renderers we're aware of - we're keeping this behaviour for now for
-			// backwards compatibility reasons.
-			typePrefix = type.substr( 0, typeColonPos ) + ":";
-
-			// This is the one case that actually works
-			if( typePrefix == "ai:" )
-			{
-				typePrefix = "arnold:";
-			}
+			// We don't currently know of any renderers where this is right, but
+			// it is our historic behaviour, which we are keeping until we know better.
+			usdShader.SetShaderId( pxr::TfToken( type.substr( 0, typeColonPos ) + ":" + shader->getName() ) );
+		}
+		else
+		{
+			usdShader.SetShaderId( pxr::TfToken( shader->getName() ) );
 		}
 	}
-	usdShader.SetShaderId( pxr::TfToken( typePrefix + shader->getName() ) );
 
 	return usdShader.ConnectableAPI();
 }

contrib/IECoreUSD/src/IECoreUSD/USDScene.cpp

@@ -497,7 +497,6 @@ void populateMaterial( pxr::UsdShadeMaterial &mat, const boost::container::flat_
 	for( const auto &[output, shaderNetwork] : shaders )
 	{
 		pxr::UsdShadeOutput matOutput = mat.CreateOutput( output, pxr::SdfValueTypeNames->Token );
-
 		std::string shaderContainerName = boost::replace_all_copy( output.GetString(), ":", "_" ) + "_shaders";
 		pxr::UsdGeomScope shaderContainer = pxr::UsdGeomScope::Define( mat.GetPrim().GetStage(), mat.GetPath().AppendChild( pxr::TfToken( shaderContainerName ) ) );
 		pxr::UsdShadeOutput networkOut = ShaderAlgo::writeShaderNetwork( shaderNetwork.get(), shaderContainer.GetPrim() );

contrib/IECoreUSD/test/IECoreUSD/USDSceneTest.py

@@ -4795,5 +4795,69 @@ def testMaterialTerminalFromSubgraph( self ) :
 
 		self.assertEqual( shaderNetwork.outputShader().name, "LamaSurface" )
 
+	def testOSLShaderForHDPrman( self ) :
+
+		self.addCleanup( os.environ.__delitem__, "IECOREUSD_WRITE_CONFORMANT_OSL_SHADERS" )
+
+		for conformant in True, False :
+
+			with self.subTest( conformant = conformant ) :
+
+				os.environ["IECOREUSD_WRITE_CONFORMANT_OSL_SHADERS"] = str( int( conformant ) )
+
+				fileName = os.path.join( self.temporaryDirectory(), f"testConformance{conformant}.usda" )
+
+				shaderNetwork = IECoreScene.ShaderNetwork(
+					shaders = {
+						"output" : IECoreScene.Shader( "PxrDiffuse", "ri:surface" ),
+						"texture" : IECoreScene.Shader( "Pattern/Noise", "osl:shader" ),
+						"scale" : IECoreScene.Shader( "floatAttribute", "osl:shader" ),
+					},
+					connections = [
+						( ( "scale", "out" ), ( "texture", "scale" ) ),
+						( ( "texture", "out" ), ( "output", "diffuseColor" ) ),
+					],
+					output = ( "output", "bxdf_out" ),
+				)
+
+				# Test writing to USD.
+
+				scene = IECoreScene.SceneInterface.create( fileName, IECore.IndexedIO.OpenMode.Write )
+				scene.createChild( "test" ).writeAttribute( "ri:surface", shaderNetwork, 0 )
+				del scene
+
+				stage = pxr.Usd.Stage.Open( fileName )
+
+				outputShader = pxr.UsdShade.Shader(
+					next( p for p in stage.Traverse() if p.IsA( pxr.UsdShade.Shader ) and p.GetName() == "output" )
+				)
+				self.assertEqual( outputShader.GetShaderId(), "PxrDiffuse" )
+
+				textureShader = pxr.UsdShade.Shader(
+					next( p for p in stage.Traverse() if p.IsA( pxr.UsdShade.Shader ) and p.GetName() == "texture" )
+				)
+				if conformant :
+					self.assertEqual( textureShader.GetShaderId(), "Noise" )
+				else :
+					self.assertEqual( textureShader.GetShaderId(), "osl:Pattern/Noise" )
+
+				scaleShader = pxr.UsdShade.Shader(
+					next( p for p in stage.Traverse() if p.IsA( pxr.UsdShade.Shader ) and p.GetName() == "scale" )
+				)
+				if conformant :
+					self.assertEqual( scaleShader.GetShaderId(), "floatAttribute" )
+				else :
+					self.assertEqual( scaleShader.GetShaderId(), "osl:floatAttribute" )
+
+				# Test loading back to Cortex.
+
+				scene = IECoreScene.SceneInterface.create( fileName, IECore.IndexedIO.OpenMode.Read )
+				loadedShaderNetwork = scene.child( "test" ).readAttribute( "ri:surface", 0 )
+
+				self.assertEqual( loadedShaderNetwork.getShader( "texture" ).name, "Pattern/Noise" )
+				self.assertEqual( loadedShaderNetwork.getShader( "texture" ).type, "osl:shader" )
+				self.assertEqual( loadedShaderNetwork.getShader( "scale" ).name, "floatAttribute" )
+				self.assertEqual( loadedShaderNetwork.getShader( "scale" ).type, "osl:shader" )
+
 if __name__ == "__main__":
 	unittest.main()