enable stencil testing

include/renderer_mtl/mtl_depth_stencil_cache.hpp

@@ -7,8 +7,10 @@ using namespace PICA;
 namespace Metal {
 
 struct DepthStencilHash {
-    bool depthWrite;
+    bool depthStencilWrite;
     u8 depthFunc;
+    u32 stencilConfig;
+    u16 stencilOpConfig;
 };
 
 class DepthStencilCache {
@@ -24,16 +26,45 @@ public:
     }
 
     MTL::DepthStencilState* get(DepthStencilHash hash) {
-        u8 intHash = hash.depthWrite | (hash.depthFunc << 1);
+        u64 intHash = ((u64)hash.depthStencilWrite << 56) | ((u64)hash.depthFunc << 48) | ((u64)hash.stencilConfig << 16) | (u64)hash.stencilOpConfig;
         auto& depthStencilState = depthStencilCache[intHash];
         if (!depthStencilState) {
             MTL::DepthStencilDescriptor* desc = MTL::DepthStencilDescriptor::alloc()->init();
-            desc->setDepthWriteEnabled(hash.depthWrite);
+            desc->setDepthWriteEnabled(hash.depthStencilWrite);
             desc->setDepthCompareFunction(toMTLCompareFunc(hash.depthFunc));
 
+            const bool stencilEnable = Helpers::getBit<0>(hash.stencilConfig);
+            MTL::StencilDescriptor* stencilDesc = nullptr;
+            if (stencilEnable) {
+               	const u8 stencilFunc = Helpers::getBits<4, 3>(hash.stencilConfig);
+               	const s8 reference = s8(Helpers::getBits<16, 8>(hash.stencilConfig)); // Signed reference value
+               	const u8 stencilRefMask = Helpers::getBits<24, 8>(hash.stencilConfig);
+
+               	const u32 stencilBufferMask = hash.depthStencilWrite ? Helpers::getBits<8, 8>(hash.stencilConfig) : 0;
+
+               	const u8 stencilFailOp = Helpers::getBits<0, 3>(hash.stencilOpConfig);
+               	const u8 depthFailOp = Helpers::getBits<4, 3>(hash.stencilOpConfig);
+               	const u8 passOp = Helpers::getBits<8, 3>(hash.stencilOpConfig);
+
+                stencilDesc = MTL::StencilDescriptor::alloc()->init();
+                stencilDesc->setStencilFailureOperation(toMTLStencilOperation(stencilFailOp));
+                stencilDesc->setDepthFailureOperation(toMTLStencilOperation(depthFailOp));
+                stencilDesc->setDepthStencilPassOperation(toMTLStencilOperation(passOp));
+                stencilDesc->setStencilCompareFunction(toMTLCompareFunc(stencilFunc));
+                stencilDesc->setReadMask(stencilRefMask);
+                stencilDesc->setWriteMask(stencilBufferMask);
+                // TODO: Set reference value
+
+                desc->setFrontFaceStencil(stencilDesc);
+                desc->setBackFaceStencil(stencilDesc);
+            }
+
             depthStencilState = device->newDepthStencilState(desc);
 
             desc->release();
+            if (stencilDesc) {
+                stencilDesc->release();
+            }
         }
 
         return depthStencilState;
@@ -47,7 +78,7 @@ public:
     }
 
 private:
-    std::unordered_map<u8, MTL::DepthStencilState*> depthStencilCache;
+    std::unordered_map<u64, MTL::DepthStencilState*> depthStencilCache;
 
     MTL::Device* device;
 };

include/renderer_mtl/pica_to_mtl.hpp

@@ -81,4 +81,20 @@ inline MTL::BlendFactor toMTLBlendFactor(u8 factor) {
     return MTL::BlendFactorOne;
 }
 
+inline MTL::StencilOperation toMTLStencilOperation(u8 op) {
+    switch (op) {
+    case 0: return MTL::StencilOperationKeep;
+    case 1: return MTL::StencilOperationZero;
+    case 2: return MTL::StencilOperationReplace;
+    case 3: return MTL::StencilOperationIncrementClamp;
+    case 4: return MTL::StencilOperationDecrementClamp;
+    case 5: return MTL::StencilOperationInvert;
+    case 6: return MTL::StencilOperationIncrementWrap;
+    case 7: return MTL::StencilOperationDecrementWrap;
+    default: panic("Unknown stencil operation %u", op);
+    }
+
+    return MTL::StencilOperationKeep;
+}
+
 } // namespace PICA

src/core/renderer_mtl/renderer_mtl.cpp

@@ -324,34 +324,33 @@ void RendererMTL::drawVertices(PICA::PrimType primType, std::span<const PICA::Ve
 
 	// Depth stencil
 	const u32 depthControl = regs[PICA::InternalRegs::DepthAndColorMask];
-	const bool depthWrite = regs[PICA::InternalRegs::DepthBufferWrite];
+	const bool depthStencilWrite = regs[PICA::InternalRegs::DepthBufferWrite];
-	const bool depthTestEnable = depthControl & 0x1;
+	const bool depthEnable = depthControl & 0x1;
 	const bool depthWriteEnable = Helpers::getBit<12>(depthControl);
 	const u8 depthFunc = Helpers::getBits<4, 3>(depthControl);
 	const u8 colorMask = Helpers::getBits<8, 4>(depthControl);
 	// TODO: color mask
 	// gl.setColourMask(colorMask & 0x1, colorMask & 0x2, colorMask & 0x4, colorMask & 0x8);
 
-	const u32 stencilConfig = regs[PICA::InternalRegs::StencilTest];
+	Metal::DepthStencilHash depthStencilHash{false, 1};
-	const bool stencilEnable = Helpers::getBit<0>(stencilConfig);
+	depthStencilHash.stencilConfig = regs[PICA::InternalRegs::StencilTest];
+	depthStencilHash.stencilOpConfig = regs[PICA::InternalRegs::StencilOp];
+	const bool stencilEnable = Helpers::getBit<0>(depthStencilHash.stencilConfig);
 
 	std::optional<Metal::DepthStencilRenderTarget> depthStencilRenderTarget = std::nullopt;
-	Metal::DepthStencilHash depthStencilHash{false, 1};
+	if (depthEnable) {
-	if (depthTestEnable) {
+		depthStencilHash.depthStencilWrite = depthWriteEnable && depthStencilWrite;
-		depthStencilHash.depthWrite = depthWriteEnable && depthWrite;
 		depthStencilHash.depthFunc = depthFunc;
 		depthStencilRenderTarget = getDepthRenderTarget();
 	} else {
 		if (depthWriteEnable) {
-			depthStencilHash.depthWrite = true;
+			depthStencilHash.depthStencilWrite = true;
 			depthStencilRenderTarget = getDepthRenderTarget();
 		} else if (stencilEnable) {
 			depthStencilRenderTarget = getDepthRenderTarget();
 		}
 	}
 
-	// TODO: stencil tests
-
 	// TODO: don't begin a new render pass every time
 	MTL::RenderPassDescriptor* renderPassDescriptor = MTL::RenderPassDescriptor::alloc()->init();
 	MTL::RenderPassColorAttachmentDescriptor* colorAttachment = renderPassDescriptor->colorAttachments()->object(0);