Is this strange inlining behavior of Visual C++ expected? - c++

I have this (rather useless) code:
__declspec(noinline)
int foo( char* ptr, int offset )
{
if( 5 / offset == 3 ) {
return 1;
}
if( ptr != ptr + offset ) {
return 2;
}
return 0;
}
int _tmain(int /*argc*/, _TCHAR* /*argv*/[])
{
if( foo( 0, 0 ) ) {
rand();
}
}
I compile that with optimizations on and get this disassembly:
141: __declspec(noinline)
142: int foo( char* ptr, int offset )
143: {
144: if( 5 / offset == 3 ) {
00401000 push 5
00401002 pop eax
00401003 cdq
00401004 xor ecx,ecx
00401006 idiv eax,ecx
00401008 sub eax,3
0040100B neg eax
0040100D sbb eax,eax
0040100F inc eax
145: return 1;
146: }
147: if( ptr != ptr + offset ) {
148: return 2;
149: }
150: return 0;
151: }
00401010 ret
152:
153: int _tmain(int /*argc*/, _TCHAR* /*argv*/[])
154: {
155: if( foo( 0, 0 ) ) {
00401011 call foo (401000h)
00401016 test eax,eax
00401018 je wmain+0Fh (401020h)
156: rand();
0040101A call dword ptr [__imp__rand (4020A0h)]
157: }
158: }
00401020 xor eax,eax
00401022 ret
The compiler preserved a foo() function call, but compiled foo() by propagating parameters known at compile time into the function body and optimizing the code. It even emitted
warning C4723: potential divide by 0
Is that expected behavior of Visual C++?

I guess so. You told it not to inline the function, but you didn't say it couldn't modify the function based on how it is used. It can see that the function is only ever called as foo(0,0), so why shouldn't it optimize the function for that?
try inserting a call to something like foo(1,2) in addition to the (0,0) call and see what happens.

Related

How to output information in Visual Studio by assembler

I try to use assembler and when I call int 0x80 my program crash. What I should do if I want to output some information in console by assembler in my C++ code?
#include <iostream>
int main()
{
char *msg = "Hello";
__asm
{
mov eax, 4;
mov ebx, 1;
mov ecx, msg;
mov edx, 5;
//int 0x80;
}
system("pause");
return 0;
}
I was found some intresting method to output Hello world in Inline ASM using Visual Studio C++.
char* hi = "Hello World\n";
char* text = "%s";
__asm
{
mov eax, hi; // load C pointer variable from memory
push eax; // function args on the stack with rightmost highest
mov eax, text;
push eax;
call DWORD ptr printf; // indirect call to DLL function
pop eax; // clean up the stack
pop eax; // with these 2 dummy pops
}
More about it is describe in this article: http://rodrigosavage.blogspot.com/2010/07/hello-world-with-inline-asm.html?m=1

how can I change modular code to OOP design?

I am trying to porting this sample of self-mutation of c to c++, but i am having problems with pointers to make it work this example for c++ OOP. I have changed this the part of displaying the data , but how can I change the rest of pointers?
aa.h
void aa::display(std::string data){
const char* str;
str = data.data();
std::cout << str << std::endl;
}
void aa::printfFunctionStub() {}
void aa::enc(DWORD dwAddress, DWORD dwSize) {
__asm {
mov ecx, dwAddress
add ecx, dwSize
mov eax, dwAddress
C_loop :
xor byte ptr ds : [eax], 0x5A
inc eax
cmp eax, ecx
jl C_loop;
}
}
aa.cpp
class aa{
public:
void debugger();
bool IsVmRunning();
void sandbox();
void foo(void);
void display(std::string data);
void printfFunctionStub();
void enc(DWORD dwAddress, DWORD dwSize);
};
main.cpp
int main(){
DWORD dwPrintFunctionSize = 0, dwOldProtect;
DWORD *fA = NULL, *fB = NULL;
// Obtain the addresses for the functions so we can calculate size.
fA = (DWORD *)&printfFunction;
fB = (DWORD *)&printfFunctionStub;
// Get total function size
dwPrintFunctionSize = (fB - fA);
// Test the function
aa.display("Hello A!\n");
// We need to give ourselves access to modifify data at the given address
VirtualProtect(fA, dwPrintFunctionSize, PAGE_READWRITE, &dwOldProtect);
enc((DWORD)fA, dwPrintFunctionSize); // XOR encrypt the function
enc((DWORD)fA, dwPrintFunctionSize); // XOR decrypt the function
// Restore the old protection
VirtualProtect(fA, dwPrintFunctionSize, dwOldProtect, NULL);
// Test the function
aa.display("Hello B!\n");
_getch();
return 0;
}

Monitoring opcode

I am trying to monitor the assembly instructions of a functions opcode. I get the functions size in bytes by subtracting a stub address from the functions in memory. I am currently only looking for the mov instruction. When I display the currentByte, it outputs only Ú, which has a hex value of 0xDA, which is FIADD in assembly http://ref.x86asm.net/coder32.html#xDA Why does no mov instruction display?
#include <iostream>
#include <Windows.h>
#include <ctime>
#include <vector>
#define PUSH 0x50
#define POP 0x58
#define MOV 0xB8
#define NOP 0x90
#define ADD 0x01
#define AND 0x21
#define XOR 0x31
#define OR 0x09
#define SBB 0x19
#define SUB 0x29
using namespace std;
int add(int x, int y)
{
int result;
__asm
{
mov eax, x
add eax, y
mov result, eax
xor eax, eax
}
return result;
}
void stub() { return; }
DWORD GetFunctionSize(DWORD* functionStartAddress, DWORD* stub)
{
DWORD dwOldProtect;
DWORD *func, *stubAddr;
func = (DWORD*)functionStartAddress;
stubAddr = (DWORD*)stub;
DWORD size = func - stubAddr;
VirtualProtect(func, size, PAGE_EXECUTE_READWRITE, &dwOldProtect);
return size;
}
void GetCurrentByte(PVOID function)
{
vector<PBYTE> currByte;
PBYTE pCurrentByte = (PBYTE)function;
if (*pCurrentByte == MOV)
{
cout << "MOV instr.\n";
}
cout << *pCurrentByte;
currByte.push_back(pCurrentByte);
}
int main()
{
DWORD size = GetFunctionSize((DWORD*)&add, (DWORD*)&stub);
for (int i = 0; i < size; i++)
{
GetCurrentByte(add);
}
system("pause");
return 0;
}
Why does no mov instruction display?
If you are in debugging mode, you need to know that you are passing a wrong address to the GetCurrentDate(PVOID), thats mean you are reading bytes from a wrong address and there is another few mistakes, to solve this issue follow those steps :
Firstly, the code bytes generated from :
mov eax, x // code bytes: 8B 45 08
mov result, eax // code bytes: 89 45 FC
0x8B and 0x89 are the values that you should look for inside your add(int, int) function.
secondly, to get the address of the first byte of your add(int, int) function i suggest to use this function :
#define ASM_CALL 0x000000E8
#define ASM_JMP 0x000000E9
#define ASM_CALL_SIZE 0x00000001
#define ASM_CALL_FULL_SIZE 0x00000005
DWORD GetFuncAddress(DWORD funcAddress)
{
BYTE calledAddress = *(BYTE*)funcAddress;
while (calledAddress == ASM_CALL || calledAddress == ASM_JMP) {
funcAddress = funcAddress + *(DWORD*)(funcAddress + ASM_CALL_SIZE) + ASM_CALL_FULL_SIZE;
calledAddress = *(BYTE*)funcAddress;
}
return funcAddress; // The address of the first byte of the function.
}
thirdly, i suggest an optimization inside your GetFunctionSize(DOWRD), as you know that your add function ends with a single return :
return result; // code bytes: C3
why not just loop throw the bytes of the add function, so when you find a byte equivalente to 0xC3, you will end up with the exact size of your function (in bytes), this code will make things clear:
#define ASM_RET 0xC3
SIZE_T GetFunctionSize(DWORD functionAddress)
{
SIZE_T funcSize = 0;
// Loop thru func's bytes, and breaks when return byte found.
while (*((PBYTE)functionAddress++) != RET)
funcSize++;
return funcSize;
}
fourthly, the GetCurrentByte(PVOID) function needs some maintenance, so i suggest :
#define ASM_MOV1 0x8B
#define ASM_MOV2 0x89
VOID GetCurrentByte(DWORD functionAddress, UINT &index)
{
BYTE tempByte = *((PBYTE)functionAddress + index);
// search for bytes which contains a mov instruction:
if (tempByte == ASM_MOV1 || tempByte == ASM_MOV2)
cout << "MOV instr found at : " << hex << ((DWORD)functionAddress + index) << endl;
}
finally, the full code will be like this :
#include <iostream>
#include <Windows.h>
#define ASM_RET 0xC3
#define ASM_MOV1 0x8B
#define ASM_MOV2 0x89
#define ASM_CALL 0xE8
#define ASM_JMP 0xE9
#define ASM_CALL_SIZE 0x01
#define ASM_CALL_FULL_SIZE 0x05
using namespace std;
INT add(INT x, INT y)
{
int result;
__asm
{
mov eax, x
add eax, y
mov result, eax
xor eax, eax
}
return result;
}
DWORD GetFuncAddress(DWORD funcAddress)
{
BYTE calledAddress = *(BYTE*)funcAddress;
while (calledAddress == ASM_CALL || calledAddress == ASM_JMP) {
funcAddress = funcAddress + *(DWORD*)(funcAddress + ASM_CALL_SIZE) + ASM_CALL_FULL_SIZE;
calledAddress = *(BYTE*)funcAddress;
}
return funcAddress;
}
SIZE_T GetFunctionSize(DWORD functionAddress)
{
SIZE_T funcSize = 0;
while (*((PBYTE)functionAddress++) != ASM_RET)
{
funcSize++;
}
return funcSize;
}
VOID GetCurrentByte(DWORD functionAddress, UINT &index)
{
BYTE tempByte = *((PBYTE)functionAddress + index);
if (tempByte == ASM_MOV1 || tempByte == ASM_MOV2)
cout << "MOV instr found at : " << hex << ((DWORD)functionAddress + index) << endl;
}
INT main()
{
DWORD funcAddress = GetFuncAddress((DWORD)add); // Get func address.
SIZE_T size = GetFunctionSize(funcAddress); // Get func size (bytes).
for (UINT i = 0; i < size; i++) // loop thru the function memory block.
{
GetCurrentByte(funcAddress, i);
}
system("pause");
return 0;
}
Don't be surprised if you found many MOV instructions in your function because the compiler created them.
Amrane Abdelkader.
GetCurrentByte() looks at the first byte every time you call it. Looking at the first byte size times doesn't help you.

How do I compile inline assembly using x64 architecture with visual studio 2012?

I'm trying to compile an inline assembly code which works fine with x86 but as I learned you cannot compile inline assembly code with x64. Is there a work around for this? Any help appreciated. The code segment that I use _asm:
int vfscanf(FILE* file, const char *format, va_list argPtr)
{
size_t count = 0;
const char* p = format;
while(1)
{
char c = *(p++);
if (c == 0)
break;
if (c == '%' && (p[0] != '*' && p[0] != '%'))
++count;
}
if (count <= 0)
return 0;
int result;
_asm
{
mov esi, esp;
}
for (int i = count - 1; i >= 0; --i)
{
_asm
{
mov eax, dword ptr[i];
mov ecx, dword ptr [argPtr];
mov edx, dword ptr [ecx+eax*4];
push edx;
}
}
int stackAdvance = (2 + count) * 4;
_asm
{
mov eax, dword ptr [format];
push eax;
mov eax, dword ptr [file];
push eax;
call dword ptr [fscanf];
mov result, eax;
mov eax, dword ptr[stackAdvance];
add esp, eax;
}
return result;
}
The code segment that I call this function:
int scan(const char * _Format, ... )
{
va_list args;
va_start(args, _Format);
_interlockedbittestandset()
int result = vfscanf(_ptr, _Format, args);
va_end(args);
return result;
}

Is there a way to implement a wrapper function for fscanf?

I was trying to write a wrapper function for all file operations. But I couldn't manage to implement wrapper function for fscanf. My own function is like this:
scan(const char * _Format, ... )
{
va_list args;
va_start(args, _Format);
int result = ::fscanf(_ptr, _Format, args);
va_end(args);
return result;
}
You need to use vfscanf. See more on vfscanf.
int scan(const char * _Format, ... )
{
va_list args;
va_start(args, _Format);
int result = ::vfscanf(_ptr, _Format, args);
va_end(args);
return result;
}
Alternatively to the use to vfscanf which takes a va_list, you may use variadic template:
template <typename ... Ts>
int scan(const char* format, Ts&&... args)
{
int result = ::fscanf(_ptr, format, std::forward<Ts>(args)...);
return result;
}
For the ones who have to use an older standard than C++11 you can implement your own vfscanf function like the following:
int vfscanf(FILE* file, const char *format, va_list argPtr)
{
size_t count = 0;
const char* p = format;
while(1)
{
char c = *(p++);
if (c == 0)
break;
if (c == '%' && (p[0] != '*' && p[0] != '%'))
++count;
}
if (count <= 0)
return 0;
int result;
_asm
{
mov esi, esp;
}
for (int i = count - 1; i >= 0; --i)
{
_asm
{
mov eax, dword ptr[i];
mov ecx, dword ptr [argPtr];
mov edx, dword ptr [ecx+eax*4];
push edx;
}
}
int stackAdvance = (2 + count) * 4;
_asm
{
mov eax, dword ptr [format];
push eax;
mov eax, dword ptr [file];
push eax;
call dword ptr [fscanf];
mov result, eax;
mov eax, dword ptr[stackAdvance];
add esp, eax;
}
return result;
}
For further information.