Use X86Assembler for windows conventions

src/assembler/X86Assembler.cpp

@@ -30,6 +30,15 @@ void X86Assembler::nop() {
 	this->write8(0x90);
 }
 
+void X86Assembler::ret() {
+	this->write8(0xC3);
+}
+
+void X86Assembler::ret(uint16_t offset) {
+	this->write8(0xC2);
+	this->write16(offset);
+}
+
 void X86Assembler::encodeModRM(X86Operand op, uint8_t digit) {
 	if (op.m_type == X86Operand::Type::Register) {
 		this->write8((0b11 << 6) | (digit << 3) | regIdx(op.m_reg));

src/assembler/X86Assembler.hpp

@@ -78,6 +78,9 @@ namespace tulip::hook {
 
 		void nop();
 
+		void ret();
+		void ret(uint16_t offset);
+
 		void add(X86Register reg, uint32_t value);
 		void sub(X86Register reg, uint32_t value);
 

src/platform/DefaultConvention.cpp

@@ -1,6 +1,7 @@
+#include "../assembler/BaseAssembler.hpp"
+
 #include <Platform.hpp>
 #include <platform/DefaultConvention.hpp>
-#incude "../assembler/BaseAssembler.hpp"
 
 using namespace tulip::hook;
 

src/platform/MacosGenerator.cpp

@@ -133,126 +133,6 @@ std::vector<uint8_t> MacosHandlerGenerator::handlerBytes(uint64_t address) {
 
 std::string MacosHandlerGenerator::handlerString() {
 	std::ostringstream out;
-	// keystone uses hex by default
-	out << std::hex;
-	out << R"ASM(
-	nop
-	nop
-	nop
-	nop
-	nop
-	nop
-	nop
-	nop
-)ASM";
-	out << m_metadata.m_convention->generateIntoDefault(m_metadata.m_abstract) << "\n ";
-
-	out << R"ASM(
-
-	; preserve registers
-	sub rsp, 0xb8
-
-	mov [rsp + 0xa8], r9
-	mov [rsp + 0xa0], r8
-	mov [rsp + 0x98], rcx
-	mov [rsp + 0x90], rdx
-	mov [rsp + 0x88], rsi
-	mov [rsp + 0x80], rdi
-	movaps [rsp + 0x70], xmm7
-	movaps [rsp + 0x60], xmm6
-	movaps [rsp + 0x50], xmm5
-	movaps [rsp + 0x40], xmm4
-	movaps [rsp + 0x30], xmm3
-	movaps [rsp + 0x20], xmm2
-	movaps [rsp + 0x10], xmm1
-	movaps [rsp + 0x00], xmm0
-
-	; preserve the original return
-	mov rax, [rsp + 0xb8]
-
-	; set the new return
-	lea rdi, [rip + _handlerCont]
-	mov [rsp + 0xb8], rdi
-
-	; set the parameters
-	mov rdi, [rip + _content]
-	mov rsi, rax
-
-	; call the pre handler, incrementing
-	mov rax, [rip + _handlerPre]
-	call rax
-
-	; recover registers
-	mov r9, [rsp + 0xa8]
-	mov r8, [rsp + 0xa0]
-	mov rcx, [rsp + 0x98]
-	mov rdx, [rsp + 0x90]
-	mov rsi, [rsp + 0x88]
-	mov rdi, [rsp + 0x80]
-	movaps xmm7, [rsp + 0x70]
-	movaps xmm6, [rsp + 0x60]
-	movaps xmm5, [rsp + 0x50]
-	movaps xmm4, [rsp + 0x40]
-	movaps xmm3, [rsp + 0x30]
-	movaps xmm2, [rsp + 0x20]
-	movaps xmm1, [rsp + 0x10]
-	movaps xmm0, [rsp + 0x00]
-
-	add rsp, 0xb8
-
-
-	; call the func
-	jmp rax
-
-_handlerCont:
-	sub rsp, 0x8
-
-	; preserve the return values
-	sub rsp, 0x38
-
-	mov [rsp + 0x28], rdx
-	mov [rsp + 0x20], rax
-	movaps [rsp + 0x10], xmm1
-	movaps [rsp + 0x00], xmm0
-
-	; call the post handler, decrementing
-	mov rax, [rip + _handlerPost]
-	call rax
-
-	; recover the original return
-	mov [rsp + 0x38], rax
-
-	; recover the return values
-	mov rdx, [rsp + 0x28]
-	mov rax, [rsp + 0x20]
-	movaps xmm1, [rsp + 0x10]
-	movaps xmm0, [rsp + 0x00]
-
-	add rsp, 0x38
-	
-	; done!
-)ASM";
-
-	out << m_metadata.m_convention->generateDefaultCleanup(m_metadata.m_abstract);
-
-	out << R"ASM(
-_handlerPre: 
-	dq 0x)ASM"
-		<< reinterpret_cast<uint64_t>(preHandler);
-
-	out << R"ASM(
-_handlerPost: 
-	dq 0x)ASM"
-		<< reinterpret_cast<uint64_t>(postHandler);
-
-	out << R"ASM(
-_content: 
-	dq 0x)ASM"
-		<< reinterpret_cast<uint64_t>(m_content);
-
-	out << R"ASM(
-_originalReturn: 
-	dq 0x0)ASM";
 
 	return out.str();
 }
@@ -279,7 +159,6 @@ std::vector<uint8_t> MacosHandlerGenerator::trampolineBytes(uint64_t address, si
 
 std::string MacosHandlerGenerator::trampolineString(size_t offset) {
 	std::ostringstream out;
-	out << "jmp _address" << m_address << "_" << offset;
 	return out.str();
 }
 

src/platform/WindowsConvention.cpp

@@ -1,12 +1,11 @@
-#include "../assembler/x86Assembler.hpp"
+#include "../assembler/X86Assembler.hpp"
 
 #include <AbstractFunction.hpp>
 #include <Platform.hpp>
 #include <algorithm>
 #include <iostream>
 #include <optional>
 #include <platform/WindowsConvention.hpp>
-#include <sstream>
 #include <variant>
 
 using namespace tulip::hook;
@@ -59,6 +58,9 @@ class PushParameters final {
 	bool m_isCallerCleanup = false;
 
 	X86Assembler& a;
+	RegMem32 m;
+
+	using enum X86Register;
 
 	PushParameters(X86Assembler& a) :
 		a(a) {}
@@ -68,13 +70,13 @@ class PushParameters final {
 		return (type.m_size + 3) / 4 * 4;
 	}
 
-	static size_t xmmRegisterName(Register reg) {
+	static X86Register xmmRegisterName(Register reg) {
 		switch (reg) {
 			default:
-			case Register::XMM0: return 0;
-			case Register::XMM1: return 1;
-			case Register::XMM2: return 2;
-			case Register::XMM3: return 3;
+			case Register::XMM0: return XMM0;
+			case Register::XMM1: return XMM1;
+			case Register::XMM2: return XMM2;
+			case Register::XMM3: return XMM3;
 		}
 	}
 
@@ -308,13 +310,10 @@ class PushParameters final {
 		}
 	}
 
-	std::string generateIntoDefault() {
-		std::ostringstream out;
-		out << std::hex;
-
+	void generateIntoDefault() {
 		// allocate space on the stack for our parameters
 		if (m_resultStackSize) {
-			out << "sub esp, 0x" << m_resultStackSize << "\n";
+			a.sub(ESP, m_resultStackSize);
 		}
 
 		// cdecl parameters are passed in reverse order; however, since we are
@@ -329,7 +328,6 @@ class PushParameters final {
 		size_t placeAt = 0x0;
 
 		for (auto& param : m_params) {
-			out << "; " << param.originalIndex << "\n";
 			// repush from stack
 			if (std::holds_alternative<Stack>(param.location)) {
 				auto offset = std::get<Stack>(param.location);
@@ -340,26 +338,30 @@ class PushParameters final {
 
 					// push the struct in the same order as it was originally on the stack
 					// + 4 is for the return address
-					out << "mov eax, [esp + 0x" << (offset + m_resultStackSize + i + 4) << "]\n";
+					a.mov(EAX, m[ESP + (offset + m_resultStackSize + i + 4)]);
 
 					// push parameters in order
-					out << "mov [esp + 0x" << (placeAt + i) << "], eax\n";
+					a.mov(m[ESP + (placeAt + i)], EAX);
 				}
 			}
 			// repush from register
 			else {
 				switch (auto reg = std::get<Register>(param.location)) {
-					case Register::ECX: out << "mov [esp + 0x" << placeAt << "], ecx\n"; break;
-					case Register::EDX: out << "mov [esp + 0x" << placeAt << "], edx\n"; break;
+					case Register::ECX: {
+						a.mov(m[ESP + placeAt], ECX);
+					} break;
+					case Register::EDX: {
+						a.mov(m[ESP + placeAt], EDX);
+					} break;
 					// xmm registers
 					default: {
 						// double
 						if (param.type.m_size == 8) {
-							out << "movsd [esp + 0x" << placeAt << "], xmm" << xmmRegisterName(reg) << "\n";
+							a.movsd(m[ESP + placeAt], xmmRegisterName(reg));
 						}
 						// float
 						else {
-							out << "movss [esp + 0x" << placeAt << "], xmm" << xmmRegisterName(reg) << "\n";
+							a.movss(m[ESP + placeAt], xmmRegisterName(reg));
 						}
 					} break;
 				}
@@ -370,88 +372,67 @@ class PushParameters final {
 			// pushing the next parameters
 			placeAt += paramSize(param.type);
 		}
-
-		return out.str();
 	}
 
-	std::string generateDefaultCleanup() {
-		std::ostringstream out;
-		out << std::hex;
-
+	void generateDefaultCleanup() {
 		// clean up stack from the ones we added
-		out << "add esp, 0x" << m_resultStackSize << "\n";
+		a.add(ESP, m_resultStackSize);
 
 		// if the function is caller cleaned, then generateOriginalCleanup
 		// or the original GD function cleans it up
 		if (m_isCallerCleanup) {
-			out << "ret\n";
+			a.ret();
 		}
 		// otherwise clean up stack using ret
 		// some of the original parameters may be passed through registers so the
 		// original's stack size may be smaller
 		else {
-			out << "ret 0x" << m_originalStackSize << "\n";
+			a.ret(m_originalStackSize);
 		}
-
-		return out.str();
 	}
 
-	std::string generateIntoOriginal() {
-		std::stringstream out;
-		out << std::hex;
-
+	void generateIntoOriginal() {
 		if (m_originalStackSize) {
-			out << "sub esp, 0x" << m_originalStackSize << "\n";
+			a.sub(ESP, m_originalStackSize);
 		}
 
 		for (auto& param : m_params) {
-			out << "; a param\n";
-			out << "; resultLocation is 0x" << param.resultLocation << "\n";
 			auto const resultOffset = m_originalStackSize + param.resultLocation + 4;
 			if (std::holds_alternative<Register>(param.location)) {
 				auto const reg = std::get<Register>(param.location);
 				switch (reg) {
 					case Register::ECX: {
-						out << "mov ecx, [esp + 0x" << resultOffset << "]\n";
+						a.mov(ECX, m[ESP + resultOffset]);
 					} break;
 					case Register::EDX: {
-						out << "mov edx, [esp + 0x" << resultOffset << "]\n";
+						a.mov(EDX, m[ESP + resultOffset]);
 					} break;
 					default: {
-						auto const xmmIndex = xmmRegisterName(reg);
 						if (param.type.m_size == 4) {
-							out << "movss xmm" << xmmIndex << ", [esp + 0x" << resultOffset << "]\n";
+							a.movss(xmmRegisterName(reg), m[ESP + resultOffset]);
 						}
 						else {
-							out << "movsd xmm" << xmmIndex << ", [esp + 0x" << resultOffset << "]\n";
+							a.movsd(xmmRegisterName(reg), m[ESP + resultOffset]);
 						}
 					}
 				}
 			}
 			else {
-				out << "; originalLocation is 0x" << param.originalLocation << "\n";
 				auto const originalOffset = m_originalStackSize + param.originalLocation + 4;
 				for (size_t i = 0; i < param.type.m_size; i += 4) {
-					out << "mov eax, [esp + 0x" << (resultOffset + i) << "]\n";
-					out << "mov [esp + 0x" << (param.originalLocation + i) << "], eax\n";
+					a.mov(EAX, m[ESP + (resultOffset + i)]);
+					a.mov(m[ESP + (originalOffset + i)], EAX);
 				}
 			}
 		}
-
-		return out.str();
 	}
 
-	std::string generateOriginalCleanup() {
-		std::ostringstream out;
-		out << std::hex;
-
+	void generateOriginalCleanup() {
 		if (m_isCallerCleanup) {
 			// for mat: comment this to make your tests work
-			out << "add esp, 0x" << m_originalStackSize << "\n";
+			a.add(ESP, m_originalStackSize);
 		}
-		out << "ret\n";
-
-		return out.str();
+		a.ret();
 	}
 };