Find Two-byte Illegal Opcodes for x86-64 - c++

Question

My goal is to find what two-byte opcodes generate an illegal instruction exception.
For example, opcodes 0F 0B UD2 raises an invalid opcode exception. The UD2 instruction is provided for software testing to explicitly generate an invalid opcode.
Warning Snake oil code ahead as I'm not familiar with Windows internals.
The code below allocates a 4K page with read/write/execute permissions and using UD2 as a starting point it tries to determine all the possible two-byte opcodes.
First, it copies the two-byte opcodes to the last two bytes of the 4K page
then executes them and checks for the exception code.
I figured that executing the last two page bytes would either
Generate an illegal exception EXCEPTION_ILLEGAL_INSTRUCTION with exactly two bytes.
Generate an access violation EXCEPTION_ACCESS_VIOLATION when extending beyond the 4K page.
Running the code below shows interesting instructions plus many unknowns too:
Illegal opcodes 0x0f 0x0b (error 0xc000001d)
ud2 - Generates an invalid opcode.
Illegal opcodes 0x0f 0x37 (error 0xc000001d)
getsec - Exit authenticated code execution mode.
Illegal opcodes 0x0f 0xaa (error 0xc000001d)
rsm - Resume operation of interrupted program.
Question
The hack'ish code runs fine in this opcode range
Executing opcodes 0x0f 0x0b ... Executing opcodes 0x0f 0xcb
until it encounters these two opcodes
0x0f 0xcc bswap esp
It seems anything that manipulates the stack pointer causes issues whereby it's stuck at this point (clicking Continue just repeats the message)
I've tried moving the opcode execution into its own thread since they have their own stack, but that didn't help!
Is there a way to preserve the stack pointers RSP and RBP or maybe there's a simple fix to resolve it?
(Built using M$ Visual C++ 2019)
#include <windows.h>
#include <stdio.h>
#include <string.h>
#include <string.h>
#include <intrin.h>
// The UD2 (0x0F, 0x0B) instruction is guaranteed to generate an invalid opcode exception.
DWORD InstructionResult;
void ExecuteOpcodes(LPVOID mem)
{
__try
{
// Execute opcodes...
((void(*)())((unsigned char*)mem + 0xFFE))();
}
__except (EXCEPTION_EXECUTE_HANDLER)
{
InstructionResult = GetExceptionCode();
}
}
int main()
{
LPVOID mem = VirtualAlloc(NULL, 2, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
DWORD oldProtect = VirtualProtect(mem, 2, PAGE_EXECUTE_READWRITE, &oldProtect);
// Start searching at the UD2 (0x0F, 0x0B) instruction which is guaranteed to generate an invalid opcode exception.
for (int i = 15; i <= 255; i++)
{
for (int j = 11; j <= 255; j++)
{
// Write two byte opcodes at the 4K page end.
*((unsigned char*)mem + 0xFFE) = i;
*((unsigned char*)mem + 0xFFF) = j;
printf("Executing opcodes 0x%02x 0x%02x\n",i,j);
HANDLE hThread = CreateThread(0, 0, (LPTHREAD_START_ROUTINE)ExecuteOpcodes, mem, 0, 0);
WaitForSingleObject(hThread, INFINITE);
CloseHandle(hThread);
if (InstructionResult == EXCEPTION_ILLEGAL_INSTRUCTION)
{
printf("Illegal opcodes 0x%02x 0x%02x (error 0x%08x)\n", i, j, InstructionResult);
}
}
}
VirtualFree(mem, 0, MEM_RELEASE);
return 0;
}
UPDATE
Based upon the good answer about creating a child process, I've pasted the updated code here: pastebin.com/j3NkL44q
#define _CRT_SECURE_NO_WARNINGS
#include <windows.h>
#include <stdio.h>
// Array of illegal single-byte opcodes
int IllegalSingleByteOpcodes[] = { 0x06 ,0x07, 0x0e, 0x16, 0x17, 0x1e, 0x1f, 0x27, 0x2f, 0x37, 0x3f, 0x60, 0x61, 0xce, 0xd6 };
// Check if a given opcode is illegal
bool is_illegal_opcode(int opcode) {
for (size_t i = 0; i < sizeof(IllegalSingleByteOpcodes) / sizeof(IllegalSingleByteOpcodes[0]); i++) {
if (IllegalSingleByteOpcodes[i] == opcode) {
return true;
}
}
return false;
}
// Create and wait for a process with the given opcodes
void create_and_wait_for_process(int opcode1, int opcode2) {
// Set up startup and process info
STARTUPINFO si;
PROCESS_INFORMATION pi;
ZeroMemory(&si, sizeof(si));
si.cb = sizeof(si);
ZeroMemory(&pi, sizeof(pi));
// Create command line string
char cmdline[256];
snprintf(cmdline, sizeof(cmdline), "IllegalOpcodes.exe %d %d", opcode1, opcode2);
// Create process
if (!CreateProcess(NULL, cmdline, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) {
printf("CreateProcess failed (%d).\n", GetLastError());
exit(1);
}
// Wait for process to finish
if (WaitForSingleObject(pi.hProcess, 1000) != WAIT_OBJECT_0) {
TerminateProcess(pi.hProcess, 0);
}
// Clean up handles
CloseHandle(pi.hProcess);
CloseHandle(pi.hThread);
}
int main() {
// Iterate through all possible opcode pairs
for (int opcode1 = 0; opcode1 <= 255; opcode1++) {
// Skip illegal opcodes
if (is_illegal_opcode(opcode1)) {
printf("\nSkipping Illegal Opcode 0x%02x ...\n", opcode1);
continue;
}
printf("\nChecking Opcode 0x%02x ...\n", opcode1);
for (int opcode2 = 0; opcode2 <= 255; opcode2++) {
create_and_wait_for_process(opcode1, opcode2);
}
}
return 0;
}
------------------- IllegalOpcodes.cpp -------------------
#include <windows.h>
#include <stdio.h>
// Offset to write opcodes at the 4K page end
#define CODE_PAGE_END_OFFSET 0xFFE
// Write two byte opcodes at the 4K page end and execute them
void write_and_execute_opcodes(LPVOID code_page_mem, int opcode1, int opcode2)
{
*((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET) = opcode1;
*((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET + 1) = opcode2;
__try
{
// Execute opcodes...
((void(*)())((unsigned char*)code_page_mem + CODE_PAGE_END_OFFSET))();
}
__except (EXCEPTION_EXECUTE_HANDLER)
{
switch (GetExceptionCode()) {
case EXCEPTION_ILLEGAL_INSTRUCTION:
printf("{0x%02x,0x%02x},", opcode1, opcode2);
break;
default:
// Ignore other exceptions
break;
}
}
}
int main(int argc, char* const argv[])
{
int opcode1 = atoi(argv[1]);
int opcode2 = atoi(argv[2]);
LPVOID code_page_mem = VirtualAlloc(NULL, 2, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
DWORD old_protect = VirtualProtect(code_page_mem, 2, PAGE_EXECUTE_READWRITE, &old_protect);
write_and_execute_opcodes(code_page_mem, opcode1, opcode2);
VirtualFree(code_page_mem, 0, MEM_RELEASE);
return 0;
}
It now includes checks to avoid these single byte illegal opcodes
32-bit Legal
06 push es
07 pop es
0e push cs
16 push ss
17 pop ss
1e push ds
1f pop ds
27 daa
2f das
37 aaa
3f aas
60 pushad
61 popad
ce into
d6 ??? <--- http://ref.x86asm.net/coder64.html#xD6
64-bit Illegal
06 ???
07 ???
0e ???
16 ???
17 ???
1e ???
1f ???
27 ???
2f ???
37 ???
3f ???
60 ???
61 ???
ce ???
d6 ???

Answer 1

maybe there's a simple fix to resolve it?
The UNIX-standard way to resolve this is to do all the test execution in a child process.
When I last worked on Windows 15 years ago, creating a child process was very expensive (slow). But since you have fewer that 64K byte combinations to try, even a slow mechanism will get you all the answers in at most a few hours.