Merge pull request #551 from wheremyfoodat/nyom

PICA Shader -> GLSL recompiler work

.github/workflows/Linux_AppImage_Build.yml

@@ -16,7 +16,7 @@ jobs:
     # well on Windows or Mac.  You can convert this to a matrix build if you need
     # cross-platform coverage.
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
@@ -33,11 +33,11 @@ jobs:
        sudo ./llvm.sh 17
        
     - name: Setup Vulkan SDK
-      run: |
-       wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
-       sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-focal.list http://packages.lunarg.com/vulkan/lunarg-vulkan-focal.list
-       sudo apt update
-       sudo apt install vulkan-sdk
+      uses: humbletim/setup-vulkan-sdk@v1.2.0
+      with:
+        vulkan-query-version: latest
+        vulkan-use-cache: true
+        vulkan-components: Vulkan-Headers, Vulkan-Loader, SPIRV-Tools, Glslang
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

.github/workflows/Qt_Build.yml

@@ -96,7 +96,7 @@ jobs:
         path: 'Alber.zip'
 
   Linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
@@ -105,7 +105,7 @@ jobs:
 
     - name: Install misc packages
       run: |
-       sudo apt-get update && sudo apt install libx11-dev libgl1-mesa-glx mesa-common-dev libfuse2 libwayland-dev
+       sudo apt-get update && sudo apt install libx11-dev libgl1-mesa-glx mesa-common-dev libfuse2 libwayland-dev libgl1-mesa-dev
        sudo add-apt-repository -y ppa:savoury1/qt-6-2
        sudo apt update
        sudo apt install qt6-base-dev qt6-base-private-dev
@@ -117,11 +117,11 @@ jobs:
        sudo ./llvm.sh 17
        
     - name: Setup Vulkan SDK
-      run: |
-       wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
-       sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-focal.list http://packages.lunarg.com/vulkan/lunarg-vulkan-focal.list
-       sudo apt update
-       sudo apt install vulkan-sdk
+      uses: humbletim/setup-vulkan-sdk@v1.2.0
+      with:
+        vulkan-query-version: latest
+        vulkan-use-cache: true
+        vulkan-components: Vulkan-Headers, Vulkan-Loader, SPIRV-Tools, Glslang
 
     - name: Configure CMake
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_COMPILER=clang-17 -DCMAKE_CXX_COMPILER=clang++-17 -DENABLE_USER_BUILD=ON -DENABLE_QT_GUI=ON

CMakeLists.txt

@@ -203,6 +203,7 @@ set(PICA_SOURCE_FILES src/core/PICA/gpu.cpp src/core/PICA/regs.cpp src/core/PICA
                       src/core/PICA/shader_interpreter.cpp src/core/PICA/dynapica/shader_rec.cpp
                       src/core/PICA/dynapica/shader_rec_emitter_x64.cpp src/core/PICA/pica_hash.cpp
                       src/core/PICA/dynapica/shader_rec_emitter_arm64.cpp src/core/PICA/shader_gen_glsl.cpp
+                      src/core/PICA/shader_decompiler.cpp
 )
 
 set(LOADER_SOURCE_FILES src/core/loader/elf.cpp src/core/loader/ncsd.cpp src/core/loader/ncch.cpp src/core/loader/3dsx.cpp src/core/loader/lz77.cpp)
@@ -253,7 +254,7 @@ set(HEADER_FILES include/emulator.hpp include/helpers.hpp include/termcolor.hpp
                  include/audio/dsp_core.hpp include/audio/null_core.hpp include/audio/teakra_core.hpp
                  include/audio/miniaudio_device.hpp include/ring_buffer.hpp include/bitfield.hpp include/audio/dsp_shared_mem.hpp
                  include/audio/hle_core.hpp include/capstone.hpp include/audio/aac.hpp include/PICA/pica_frag_config.hpp
-                 include/PICA/pica_frag_uniforms.hpp include/PICA/shader_gen_types.hpp
+                 include/PICA/pica_frag_uniforms.hpp include/PICA/shader_gen_types.hpp include/PICA/shader_decompiler.hpp
 )
 
 cmrc_add_resource_library(

include/PICA/shader.hpp

@@ -1,6 +1,8 @@
 #pragma once
 #include <algorithm>
 #include <array>
+#include <cassert>
+#include <cstddef>
 #include <cstring>
 
 #include "PICA/float_types.hpp"
@@ -90,9 +92,12 @@ class PICAShader {
   public:
 	// These are placed close to the temp registers and co because it helps the JIT generate better code
 	u32 entrypoint = 0;  // Initial shader PC
-	u32 boolUniform;
-	std::array<std::array<u8, 4>, 4> intUniforms;
+
+	// We want these registers in this order & with this alignment for uploading them directly to a UBO
+	// When emulating shaders on the GPU. Plus this alignment for float uniforms is necessary for doing SIMD in the shader->CPU recompilers.
 	alignas(16) std::array<vec4f, 96> floatUniforms;
+	alignas(16) std::array<std::array<u8, 4>, 4> intUniforms;
+	u32 boolUniform;
 
 	alignas(16) std::array<vec4f, 16> fixedAttributes;  // Fixed vertex attributes
 	alignas(16) std::array<vec4f, 16> inputs;           // Attributes passed to the shader
@@ -291,4 +296,9 @@ class PICAShader {
 
 	Hash getCodeHash();
 	Hash getOpdescHash();
-};
+};
+
+static_assert(
+	offsetof(PICAShader, intUniforms) == offsetof(PICAShader, floatUniforms) + 96 * sizeof(float) * 4 &&
+	offsetof(PICAShader, boolUniform) == offsetof(PICAShader, intUniforms) + 4 * sizeof(u8) * 4
+);

include/PICA/shader_decompiler.hpp

@@ -0,0 +1,112 @@
+#pragma once
+#include <set>
+#include <string>
+#include <tuple>
+#include <map>
+#include <vector>
+
+#include "PICA/shader.hpp"
+#include "PICA/shader_gen_types.hpp"
+
+struct EmulatorConfig;
+
+namespace PICA::ShaderGen {
+	// Control flow analysis is partially based on
+	// https://github.com/PabloMK7/citra/blob/d0179559466ff09731d74474322ee880fbb44b00/src/video_core/shader/generator/glsl_shader_decompiler.cpp#L33
+	struct ControlFlow {
+		// A continuous range of addresses
+		struct AddressRange {
+			u32 start, end;
+			AddressRange(u32 start, u32 end) : start(start), end(end) {}
+
+			// Use lexicographic comparison for functions in order to sort them in a set
+			bool operator<(const AddressRange& other) const { return std::tie(start, end) < std::tie(other.start, other.end); }
+		};
+
+		struct Function {
+			using Labels = std::set<u32>;
+
+			enum class ExitMode {
+				Unknown,       // Can't guarantee whether we'll exit properly, fall back to CPU shaders (can happen with jmp shenanigans)
+				AlwaysReturn,  // All paths reach the return point.
+				Conditional,   // One or more code paths reach the return point or an END instruction conditionally.
+				AlwaysEnd,     // All paths reach an END instruction.
+			};
+
+			u32 start;           // Starting PC of the function
+			u32 end;             // End PC of the function
+			Labels outLabels{};  // Labels this function can "goto" (jump) to
+			ExitMode exitMode = ExitMode::Unknown;
+
+			explicit Function(u32 start, u32 end) : start(start), end(end) {}
+			bool operator<(const Function& other) const { return AddressRange(start, end) < AddressRange(other.start, other.end); }
+
+			std::string getIdentifier() const { return "func_" + std::to_string(start) + "_to_" + std::to_string(end); }
+			std::string getForwardDecl() const { return "void " + getIdentifier() + "();\n"; }
+			std::string getCallStatement() const { return getIdentifier() + "()"; }
+		};
+
+		std::set<Function> functions{};
+		std::map<AddressRange, Function::ExitMode> exitMap{};
+
+		// Tells us whether analysis of the shader we're trying to compile failed, in which case we'll need to fail back to shader emulation
+		// On the CPU
+		bool analysisFailed = false;
+
+		// This will recursively add all functions called by the function too, as analyzeFunction will call addFunction on control flow instructions
+		const Function* addFunction(const PICAShader& shader, u32 start, u32 end) {
+			auto searchIterator = functions.find(Function(start, end));
+			if (searchIterator != functions.end()) {
+				return &(*searchIterator);
+			}
+
+			// Add this function and analyze it if it doesn't already exist
+			Function function(start, end);
+			function.exitMode = analyzeFunction(shader, start, end, function.outLabels);
+			
+			// This function could not be fully analyzed, report failure
+			if (function.exitMode == Function::ExitMode::Unknown) {
+				analysisFailed = true;
+				return nullptr;
+			}
+
+			// Add function to our function list
+			auto [it, added] = functions.insert(std::move(function));
+			return &(*it);
+		}
+
+		void analyze(const PICAShader& shader, u32 entrypoint);
+		Function::ExitMode analyzeFunction(const PICAShader& shader, u32 start, u32 end, Function::Labels& labels);
+	};
+
+	class ShaderDecompiler {
+		using AddressRange = ControlFlow::AddressRange;
+		using Function = ControlFlow::Function;
+
+		ControlFlow controlFlow{};
+
+		PICAShader& shader;
+		EmulatorConfig& config;
+		std::string decompiledShader;
+
+		u32 entrypoint;
+
+		API api;
+		Language language;
+
+		void compileInstruction(u32& pc, bool& finished);
+		void compileRange(const AddressRange& range);
+		void callFunction(const Function& function);
+		const Function* findFunction(const AddressRange& range);
+
+		void writeAttributes();
+
+	  public:
+		ShaderDecompiler(PICAShader& shader, EmulatorConfig& config, u32 entrypoint, API api, Language language)
+			: shader(shader), entrypoint(entrypoint), config(config), api(api), language(language), decompiledShader("") {}
+
+		std::string decompile();
+	};
+
+	std::string decompileShader(PICAShader& shader, EmulatorConfig& config, u32 entrypoint, API api, Language language);
+}  // namespace PICA::ShaderGen

src/core/PICA/shader_decompiler.cpp

@@ -0,0 +1,153 @@
+#include "PICA/shader_decompiler.hpp"
+
+#include "config.hpp"
+
+using namespace PICA;
+using namespace PICA::ShaderGen;
+using Function = ControlFlow::Function;
+using ExitMode = Function::ExitMode;
+
+void ControlFlow::analyze(const PICAShader& shader, u32 entrypoint) {
+	analysisFailed = false;
+
+	const Function* function = addFunction(shader, entrypoint, PICAShader::maxInstructionCount);
+	if (function == nullptr) {
+		analysisFailed = true;
+	}
+}
+
+ExitMode ControlFlow::analyzeFunction(const PICAShader& shader, u32 start, u32 end, Function::Labels& labels) {
+	// Initialize exit mode to unknown by default, in order to detect things like unending loops
+	auto [it, inserted] = exitMap.emplace(AddressRange(start, end), ExitMode::Unknown);
+	// Function has already been analyzed and is in the map so it wasn't added, don't analyze again
+	if (!inserted) {
+		return it->second;
+	}
+
+	// Make sure not to go out of bounds on the shader
+	for (u32 pc = start; pc < PICAShader::maxInstructionCount && pc != end; pc++) {
+		const u32 instruction = shader.loadedShader[pc];
+		const u32 opcode = instruction >> 26;
+
+		switch (opcode) {
+			case ShaderOpcodes::JMPC: Helpers::panic("Unimplemented control flow operation (JMPC)");
+			case ShaderOpcodes::JMPU: Helpers::panic("Unimplemented control flow operation (JMPU)");
+			case ShaderOpcodes::IFU: Helpers::panic("Unimplemented control flow operation (IFU)");
+			case ShaderOpcodes::IFC: Helpers::panic("Unimplemented control flow operation (IFC)");
+			case ShaderOpcodes::CALL: Helpers::panic("Unimplemented control flow operation (CALL)");
+			case ShaderOpcodes::CALLC: Helpers::panic("Unimplemented control flow operation (CALLC)");
+			case ShaderOpcodes::CALLU: Helpers::panic("Unimplemented control flow operation (CALLU)");
+			case ShaderOpcodes::LOOP: Helpers::panic("Unimplemented control flow operation (LOOP)");
+			case ShaderOpcodes::END: it->second = ExitMode::AlwaysEnd; return it->second;
+
+			default: break;
+		}
+	}
+
+	// A function without control flow instructions will always reach its "return point" and return
+	return ExitMode::AlwaysReturn;
+}
+
+void ShaderDecompiler::compileRange(const AddressRange& range) {
+	u32 pc = range.start;
+	const u32 end = range.end >= range.start ? range.end : PICAShader::maxInstructionCount;
+	bool finished = false;
+
+	while (pc < end && !finished) {
+		compileInstruction(pc, finished);
+	}
+}
+
+const Function* ShaderDecompiler::findFunction(const AddressRange& range) {
+	for (const Function& func : controlFlow.functions) {
+		if (range.start == func.start && range.end == func.end) {
+			return &func;
+		}
+	}
+
+	return nullptr;
+}
+
+void ShaderDecompiler::writeAttributes() {
+	decompiledShader += R"(
+		layout(std140) uniform PICAShaderUniforms {
+			vec4 uniform_float[96];
+			uvec4 uniform_int;
+			uint uniform_bool;
+		};
+)";
+
+	decompiledShader += "\n";
+}
+
+std::string ShaderDecompiler::decompile() {
+	controlFlow.analyze(shader, entrypoint);
+
+	if (controlFlow.analysisFailed) {
+		return "";
+	}
+
+	decompiledShader = "";
+
+	switch (api) {
+		case API::GL: decompiledShader += "#version 410 core\n"; break;
+		case API::GLES: decompiledShader += "#version 300 es\n"; break;
+		default: break;
+	}
+
+	writeAttributes();
+
+	if (config.accurateShaderMul) {
+		// Safe multiplication handler from Citra: Handles the PICA's 0 * inf = 0 edge case
+		decompiledShader += R"(
+			vec4 safe_mul(vec4 a, vec4 b) {
+				vec4 res = a * b;
+				return mix(res, mix(mix(vec4(0.0), res, isnan(rhs)), product, isnan(lhs)), isnan(res));
+			}
+		)";
+	}
+
+	// Forward declare every generated function first so that we can easily call anything from anywhere.
+	for (auto& func : controlFlow.functions) {
+		decompiledShader += func.getForwardDecl();
+	}
+
+	decompiledShader += "void pica_shader_main() {\n";
+	AddressRange mainFunctionRange(entrypoint, PICAShader::maxInstructionCount);
+	callFunction(*findFunction(mainFunctionRange));
+	decompiledShader += "}\n";
+
+	for (auto& func : controlFlow.functions) {
+		if (func.outLabels.size() > 0) {
+			Helpers::panic("Function with out labels");
+		}
+
+		decompiledShader += "void " + func.getIdentifier() + "() {\n";
+		compileRange(AddressRange(func.start, func.end));
+		decompiledShader += "}\n";
+	}
+
+	return decompiledShader;
+}
+
+void ShaderDecompiler::compileInstruction(u32& pc, bool& finished) {
+	const u32 instruction = shader.loadedShader[pc];
+	const u32 opcode = instruction >> 26;
+
+	switch (opcode) {
+		case ShaderOpcodes::DP4: decompiledShader += "dp4\n"; break;
+		case ShaderOpcodes::MOV: decompiledShader += "mov\n"; break;
+		case ShaderOpcodes::END: finished = true; return;
+		default: Helpers::warn("GLSL recompiler: Unknown opcode: %X", opcode); break;
+	}
+
+	pc++;
+}
+
+void ShaderDecompiler::callFunction(const Function& function) { decompiledShader += function.getCallStatement() + ";\n"; }
+
+std::string ShaderGen::decompileShader(PICAShader& shader, EmulatorConfig& config, u32 entrypoint, API api, Language language) {
+	ShaderDecompiler decompiler(shader, config, entrypoint, api, language);
+
+	return decompiler.decompile();
+}