Related
I am trying to write an application to capture an image from a webcam. I have written a C++ console application and whilst I can get an image, I have difficulty setting the resolution, I want to get higher resolution images and would like to be able to get 4k from a logitech brio webcam for example.
On Windows 10 with a lower resolution usb camera I was able to get 1280x720. On Windows 11 with a 4k USB camera I am only getting 640x480
Here is my code:
#include "vfw.h"
#include <cstdio>
#include <cstdint>
#pragma comment(lib, "Vfw32.lib")
int main(int argc, char** argv)
{
char device_name[100];
char filename[100];
size_t sz = 0;
char* userpath = nullptr;
strcpy_s(device_name, "");
strcpy_s(filename, userpath);
strncat_s(filename, "\\Documents\\image.bmp", 20);
// create the preview window
HWND hCam = capCreateCaptureWindow (
"FMCamCapture",
WS_CHILD | WS_VISIBLE,
0, 0, 1024, 768,
::GetDesktopWindow(), 0);
CAPDRIVERCAPS CapDrvCaps;
capDriverGetCaps(hCam, &CapDrvCaps, sizeof(CAPDRIVERCAPS));
int driver_number = 0;
int n = 1;
//Driver argument
while (n < argc)
{
if (strcmp(argv[n], "/drivernum") == 0)
{
// Set device number to specified value
if (++n < argc) {
driver_number = atoi(argv[n]);
}
else printf("Error: invalid device number");
if (driver_number <= 0)
printf("Error: invalid device number");
}
if (strcmp(argv[n], "/showdrivers") == 0)
{
// Show the list of available drivers and pause and exit
TCHAR szDeviceName[80];
TCHAR szDeviceVersion[80];
for (int wIndex = 0; wIndex < 10; wIndex++)
{
if (capGetDriverDescription(
wIndex,
szDeviceName,
sizeof(szDeviceName),
szDeviceVersion,
sizeof(szDeviceVersion)
))
{
printf("\nEach Driver may represent a different camera. The default driver is 0. Set a differnt driver to change cameras.\nAvailable Drivers:\n\%i - %s\n", wIndex, szDeviceName);
// Append name to list of installed capture drivers
// and then let the user select a driver to use.
}
}
system("pause");
return 0;
}
n++;
}
fprintf(stderr, "Saved file location: %s\n", filename);
CAPTUREPARMS captureParms;
BITMAPINFO bMInfo;
// connect to the first Driver
// for other cameras try index
// 1, 2, in place of the 0 below
if(capDriverConnect(hCam, driver_number))
{
capGetVideoFormat(hCam, &bMInfo, sizeof(BITMAPINFO));
//printf("height %ld", bMInfo.bmiHeader.biHeight);
//printf("width %ld", bMInfo.bmiHeader.biWidth);
//printf("bisize %lu", bMInfo.bmiHeader.biSizeImage);
//printf("compression %lu", bMInfo.bmiHeader.biCompression);
bMInfo.bmiHeader.biWidth = 1280;
bMInfo.bmiHeader.biHeight = 720;
bMInfo.bmiHeader.biSizeImage = 0;
//bMInfo.bmiHeader.biSizeImage = (bMInfo.bmiHeader.biHeight) * (((bMInfo.bmiHeader.biWidth * 2) * bMInfo.bmiHeader.biBitCount + 31) / 32) * 4;
bMInfo.bmiHeader.biCompression = BI_RGB;
if (capSetVideoFormat(hCam, &bMInfo, sizeof(BITMAPINFO))) {
//printf("set res success setvidformat");
}
capFileSaveDIB(hCam, filename);
//printf("Saved!");
capDriverDisconnect(hCam);
}
else
{
printf("Check camera?");
}
DestroyWindow(hCam);
return 0;
}```
I am currently trying to get the wintun driver to work with my program for simple tunneling (see: https://www.wintun.net/ ).
I successfully find and open the network device, but when it comes to registering the buffer, I get the result ERROR_INVALID_PARAMETER (87). Like I said, opening works just fine and registering is done with SYSTEM privileges (if this is not done, I get ERROR_ACCESS_DENIED (5)).
First attempt was to malloc the ring buffers, but after that did not work I looked at how OpenVPN does it (yes, it added wintun support) and they seem to do with with CreateFileMapping.
First of all, here is my struct:
typedef struct _TUN_RING {
volatile ULONG Head;
volatile ULONG Tail;
volatile LONG Alertable;
UCHAR Data[(1024 * 1024) + 0x10000];
} TUN_RING;
which is according to the docs (https://git.zx2c4.com/wintun/about/ section "Ring Layout). Also its the same as OpenVPN does.
After that I create the file mapping
send_ring_handle_ = CreateFileMapping(INVALID_HANDLE_VALUE,
nullptr,
PAGE_READWRITE,
0,
sizeof(TUN_RING),
nullptr);
recv_ring_handle_ = CreateFileMapping(INVALID_HANDLE_VALUE,
nullptr,
PAGE_READWRITE,
0,
sizeof(TUN_RING),
nullptr);
Then I create the mappings:
send_ring_ = (TUN_RING *)MapViewOfFile(send_ring_handle_,
FILE_MAP_ALL_ACCESS,
0,
0,
sizeof(TUN_RING));
recv_ring_ = (TUN_RING *)MapViewOfFile(recv_ring_handle_,
FILE_MAP_ALL_ACCESS,
0,
0,
sizeof(TUN_RING));
and finally (after impersonating the system user) trying to register it with DeviceIoControl:
TUN_REGISTER_RINGS reg_rings;
memset(®_rings, 0, sizeof(TUN_REGISTER_RINGS));
reg_rings.Send.RingSize = sizeof(TUN_RING);
reg_rings.Send.Ring = send_ring_;
reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
reg_rings.Receive.RingSize = sizeof(TUN_RING);
reg_rings.Receive.Ring = recv_ring_;
reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
DWORD len;
if (!DeviceIoControl(tun_fd_,
TUN_IOCTL_REGISTER_RINGS,
®_rings,
sizeof(reg_rings),
nullptr,
0,
&len,
nullptr))
{
printf("Could not register ring buffers (%d).", ::GetLastError());
return false;
}
Can anybody point me to where I am wrong? Like I said, with malloc instead of the file mapping the same error arieses.
I have written a complete example by now using malloc:
#include <windows.h>
#include <winioctl.h>
#include <IPHlpApi.h>
#include <ndisguid.h>
#include <TlHelp32.h>
#include <tchar.h>
#include <securitybaseapi.h>
#include <cfgmgr32.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <assert.h>
#pragma pack(push, 1)
typedef struct _TUN_PACKET_PROTO {
ULONG Size;
UCHAR Data[]; // max packet size as defined by the driver.
} TUN_PACKET_PROTO;
typedef struct _TUN_RING_PROTO {
volatile ULONG Head;
volatile ULONG Tail;
volatile LONG Alertable;
UCHAR Data[];
} TUN_RING_PROTO;
#define TUN_IOCTL_REGISTER_RINGS CTL_CODE(51820U, 0x970U, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)
#define TUN_IOCTL_FORCE_CLOSE_HANDLES CTL_CODE(51820U, 0x971U, METHOD_NEITHER, FILE_READ_DATA | FILE_WRITE_DATA)
#define WINTUN_RING_CAPACITY 0x800000
#define WINTUN_RING_TRAILING_BYTES 0x10000
#define WINTUN_MAX_PACKET_SIZE 0xffff
#define WINTUN_PACKET_ALIGN 4
/* Memory alignment of packets and rings */
#define TUN_ALIGNMENT sizeof(ULONG)
#define TUN_ALIGN(Size) (((ULONG)(Size) + ((ULONG)TUN_ALIGNMENT - 1)) & ~((ULONG)TUN_ALIGNMENT - 1))
#define TUN_IS_ALIGNED(Size) (!((ULONG)(Size) & ((ULONG)TUN_ALIGNMENT - 1)))
/* Maximum IP packet size */
#define TUN_MAX_IP_PACKET_SIZE 0xFFFF
/* Maximum packet size */
#define TUN_MAX_PACKET_SIZE TUN_ALIGN(sizeof(TUN_PACKET_PROTO) + TUN_MAX_IP_PACKET_SIZE)
/* Minimum ring capacity. */
#define TUN_MIN_RING_CAPACITY 0x20000 /* 128kiB */
/* Maximum ring capacity. */
#define TUN_MAX_RING_CAPACITY 0x4000000 /* 64MiB */
/* Calculates ring capacity */
#define TUN_RING_CAPACITY(Size) ((Size) - sizeof(TUN_RING_PROTO) - (TUN_MAX_PACKET_SIZE - TUN_ALIGNMENT))
/* Calculates ring offset modulo capacity */
#define TUN_RING_WRAP(Value, Capacity) ((Value) & (Capacity - 1))
#define IS_POW2(x) ((x) && !((x) & ((x)-1)))
typedef struct _TUN_RING {
volatile ULONG Head;
volatile ULONG Tail;
volatile LONG Alertable;
UCHAR Data[WINTUN_RING_CAPACITY + (TUN_MAX_PACKET_SIZE-TUN_ALIGNMENT)];
} TUN_RING;
typedef struct _TUN_PACKET {
ULONG Size;
UCHAR Data[WINTUN_MAX_PACKET_SIZE]; // max packet size as defined by the driver.
} TUN_PACKET;
typedef struct _TUN_REGISTER_RINGS {
struct {
ULONG RingSize;
TUN_RING *Ring;
HANDLE TailMoved;
} Send, Receive;
} TUN_REGISTER_RINGS;
#pragma pack(pop)
class regkey_t
{
public:
regkey_t(void);
regkey_t(HKEY handle);
~regkey_t(void);
void attach(HKEY handle);
void release(void);
HKEY detach(void);
operator HKEY (void) const;
HKEY &get(void);
HKEY *operator &(void);
private:
regkey_t(const regkey_t &);
regkey_t &operator = (const regkey_t &);
HKEY handle_;
};
regkey_t::regkey_t():
handle_(0)
{
}
regkey_t::regkey_t(HKEY handle):
handle_(handle)
{
}
regkey_t::~regkey_t(void)
{
release();
}
void regkey_t::attach(HKEY handle)
{
release();
handle_ = handle;
}
void regkey_t::release(void)
{
if (handle_)
{
const LONG res (::RegCloseKey(handle_));
if (res != ERROR_SUCCESS)
{
printf("Couldn't close a reg handle (%lu).\n", res);
}
handle_ = 0;
}
}
HKEY regkey_t::detach(void)
{
const HKEY result (handle_);
handle_ = 0;
return result;
}
HKEY ®key_t::get(void)
{
return handle_;
}
HKEY *regkey_t::operator &(void)
{
return &handle_;
}
regkey_t::operator HKEY(void) const
{
return handle_;
}
bool impersonate_as_system()
{
HANDLE thread_token, process_snapshot, winlogon_process, winlogon_token, duplicated_token;
PROCESSENTRY32 entry;
BOOL ret;
DWORD pid = 0;
TOKEN_PRIVILEGES privileges;
::memset(&entry, 0, sizeof(entry));
::memset(&privileges, 0, sizeof(privileges));
entry.dwSize = sizeof(PROCESSENTRY32);
privileges.PrivilegeCount = 1;
privileges.Privileges->Attributes = SE_PRIVILEGE_ENABLED;
if (!LookupPrivilegeValue(NULL, SE_DEBUG_NAME, &privileges.Privileges[0].Luid))
{
return false;
}
if (!ImpersonateSelf(SecurityImpersonation))
{
return false;
}
if (!OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES, FALSE, &thread_token))
{
RevertToSelf();
return false;
}
if (!AdjustTokenPrivileges(thread_token, FALSE, &privileges, sizeof(privileges), NULL, NULL))
{
CloseHandle(thread_token);
RevertToSelf();
return false;
}
CloseHandle(thread_token);
process_snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
if (process_snapshot == INVALID_HANDLE_VALUE)
{
RevertToSelf();
return false;
}
for (ret = Process32First(process_snapshot, &entry); ret; ret = Process32Next(process_snapshot, &entry))
{
if (::strcmp(entry.szExeFile, "winlogon.exe") == 0)
{
pid = entry.th32ProcessID;
break;
}
}
CloseHandle(process_snapshot);
if (!pid)
{
RevertToSelf();
return false;
}
winlogon_process = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
if (!winlogon_process)
{
RevertToSelf();
return false;
}
if (!OpenProcessToken(winlogon_process, TOKEN_IMPERSONATE | TOKEN_DUPLICATE, &winlogon_token))
{
CloseHandle(winlogon_process);
RevertToSelf();
return false;
}
CloseHandle(winlogon_process);
if (!DuplicateToken(winlogon_token, SecurityImpersonation, &duplicated_token))
{
CloseHandle(winlogon_token);
RevertToSelf();
return false;
}
CloseHandle(winlogon_token);
if (!SetThreadToken(NULL, duplicated_token))
{
CloseHandle(duplicated_token);
RevertToSelf();
return false;
}
CloseHandle(duplicated_token);
return true;
}
std::string get_instance_id(uint32_t device_index)
{
const std::string key_name("SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}");
std::string device_id("");
regkey_t adapters;
DWORD ret = ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, key_name.c_str(), 0, KEY_READ, &adapters);
if (ret != ERROR_SUCCESS)
{
printf("Could not open registry key %s (%d).\n", key_name.c_str(), ret);
return device_id;
}
DWORD sub_keys(0);
ret = ::RegQueryInfoKey(adapters, NULL, NULL, NULL, &sub_keys, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
if (ret != ERROR_SUCCESS)
{
printf("Could not get info from %s (%d).\n", key_name.c_str(), ret);
return device_id;
}
if (sub_keys <= 0)
{
printf("Wrong registry key %s.\n", key_name.c_str());
return device_id;
}
if (device_index >= sub_keys)
{
return device_id;
}
uint32_t index(0);
for (DWORD i = 0; i < sub_keys; i++)
{
const uint32_t max_key_length = 255;
TCHAR key[max_key_length];
DWORD keylen(max_key_length);
// Get the adapter name
ret = ::RegEnumKeyEx(adapters, i, key, &keylen, NULL, NULL, NULL, NULL);
if (ret != ERROR_SUCCESS)
{
continue;
}
// Append it to NETWORK_ADAPTERS and open it
regkey_t device;
const std::string new_key(key_name + "\\" + std::string(key));
ret = ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, new_key.c_str(), 0, KEY_READ, &device);
if (ret != ERROR_SUCCESS)
{
continue;
}
TCHAR data[256];
DWORD len(sizeof(data));
ret = ::RegQueryValueEx(device, "ComponentId", NULL, NULL, (LPBYTE)data, &len);
if (ret != ERROR_SUCCESS)
{
continue;
}
std::string device_name("wintun");
if (::_tcsnccmp(data, device_name.c_str(), sizeof(TCHAR) * device_name.length()) == 0)
{
if (device_index != index)
{
index++;
continue;
}
DWORD type;
len = sizeof(data);
ret = ::RegQueryValueEx(device, "DeviceInstanceID", NULL, &type, (LPBYTE)data, &len);
if (ret != ERROR_SUCCESS)
{
printf("Could not get info from %s (%d).\n", key_name.c_str(), ret);
}
device_id = data;
break;
}
}
return device_id;
}
bool open_tun_device()
{
HANDLE tun_fd_ = INVALID_HANDLE_VALUE;
std::string device_id;
uint32_t device_index;
{
TCHAR *interface_list = nullptr;
for (device_index = 0; device_index < 256; ++device_index)
{
device_id = get_instance_id(device_index);
if (device_id.empty())
{
continue;
}
CONFIGRET status = CR_SUCCESS;
// This loop is recommended as "robust code" by MSDN. See the Remarks of CM_Get_Device_Interface_list.
do
{
DWORD required_chars(0);
if ((status = ::CM_Get_Device_Interface_List_Size(&required_chars,
(LPGUID)&GUID_DEVINTERFACE_NET,
(char *)device_id.c_str(),
CM_GET_DEVICE_INTERFACE_LIST_PRESENT)) != CR_SUCCESS || !required_chars)
{
break;
}
assert(required_chars > 0);
interface_list = (TCHAR *)::malloc(sizeof(TCHAR) * required_chars);
status = ::CM_Get_Device_Interface_List((LPGUID)&GUID_DEVINTERFACE_NET,
(char *)device_id.c_str(),
interface_list,
required_chars,
CM_GET_DEVICE_INTERFACE_LIST_PRESENT);
if (status == CR_SUCCESS)
{
break;
}
::free(interface_list);
interface_list = nullptr;
} while(status == CR_BUFFER_SMALL);
if (interface_list)
{
break;
}
}
if (!interface_list)
{
printf("Could not find wintun interface.\n");
return false;
}
else
{
tun_fd_ = ::CreateFile(interface_list,
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr,
OPEN_EXISTING, 0, nullptr);
}
::free(interface_list);
}
if (!tun_fd_ || tun_fd_ == INVALID_HANDLE_VALUE)
{
printf("Could not open wintun device.\n");
return false;
}
printf("Opened wintun device.\n");
::Sleep(1000);
TUN_RING * send_ring_ = (TUN_RING *)::malloc(sizeof(TUN_RING));
TUN_RING * recv_ring_ = (TUN_RING *)::malloc(sizeof(TUN_RING));
if (!recv_ring_ || !send_ring_)
{
printf("Could not malloc.\n");
return false;
}
::memset(send_ring_, 0, sizeof(*send_ring_));
::memset(recv_ring_, 0, sizeof(*recv_ring_));
recv_ring_->Alertable = TRUE;
recv_ring_->Head = 0;
recv_ring_->Tail = 0;
send_ring_->Alertable = TRUE;
send_ring_->Head = 0;
send_ring_->Tail = 0;
HANDLE send_event = ::CreateEvent(0, FALSE, FALSE, 0);
HANDLE recv_event = ::CreateEvent(0, FALSE, FALSE, 0);
// register the rings
if (impersonate_as_system())
{
TUN_REGISTER_RINGS reg_rings;
::memset(®_rings, 0, sizeof(TUN_REGISTER_RINGS));
reg_rings.Send.RingSize = sizeof(TUN_RING);
reg_rings.Send.Ring = send_ring_;
reg_rings.Send.TailMoved = send_event;
reg_rings.Receive.RingSize = sizeof(TUN_RING);
reg_rings.Receive.Ring = recv_ring_;
reg_rings.Receive.TailMoved = recv_event;
int send_capacity = TUN_RING_CAPACITY(reg_rings.Send.RingSize);
if ((send_capacity < TUN_MIN_RING_CAPACITY || send_capacity > TUN_MAX_RING_CAPACITY ||
!IS_POW2(send_capacity) || !reg_rings.Send.TailMoved || !reg_rings.Send.Ring))
{
printf("Fuck this shit I am out...\n");
}
DWORD len;
DWORD fuckyou = 0;
if (!::DeviceIoControl(tun_fd_, TUN_IOCTL_FORCE_CLOSE_HANDLES,
&fuckyou, sizeof(fuckyou), nullptr, 0, &len, nullptr))
{
printf("Error releasing handles (%d).\n", ::GetLastError());
}
if (!::DeviceIoControl(tun_fd_,
TUN_IOCTL_REGISTER_RINGS,
®_rings,
sizeof(reg_rings),
nullptr,
0,
&len,
nullptr))
{
printf("Could not register ring buffers (%d).\n", ::GetLastError());
::Sleep(10000);
RevertToSelf();
return false;
}
::Sleep(10000);
RevertToSelf();
}
else
{
printf("Could not elevate to SYSTEM\n");
return false;
}
return true;
}
int main()
{
if (!open_tun_device())
{
printf("Experiment failed.\n");
}
printf("Size TUNRING: %d (%d)\n", sizeof(TUN_RING), 0x800000 + 0x010000 + 0x0C);
printf("Capacity: %d\n", TUN_RING_CAPACITY(sizeof(TUN_RING)));
if (!IS_POW2(TUN_RING_CAPACITY(sizeof(TUN_RING))))
{
printf("Shit gone wrong...\n");
}
return 0;
}
Please make sure to RUN THIS ELEVATED or you will get error 5 ERROR_ACCESS_DENIED.
I can see a difference in your code when registering rings.
You are doing:
reg_rings.Send.RingSize = sizeof(TUN_RING);
reg_rings.Receive.RingSize = sizeof(TUN_RING);
While the docs says:
Send.RingSize, Receive.RingSize: Sizes of the rings (sizeof(TUN_RING) + capacity + 0x10000, as above)
Your ring is sizeof(TUN_RING) + UCHAR[(1024 * 1024) + 0x10000]
I guess it can't accept a ring that has no data space?
Sorry, I see your TUN_RING includes de data...
May be the events aren't good:
If an event is created from a service or a thread that is impersonating a different user, you can either apply a security descriptor to the event when you create it, or change the default security descriptor for the creating process by changing its default DACL
reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
You seem to be using the default DACL.
There may be aligning problems. If malloc isn't returning an aligned address for your buffer (as may be in debug mode, because there are memory management bytes) your Data member for the packet could be not aligned.
You can check the alignment against the address:
template <unsigned int alignment>
struct IsAligned
{
static_assert((alignment & (alignment - 1)) == 0, "Alignment must be a power of 2");
static inline bool Value(const void * ptr)
{
return (((uintptr_t)ptr) & (alignment - 1)) == 0;
}
};
std::cout << IsAligned<32>::Value(ptr + i) << std::endl;
Giving the first packet address &(TUN_RING.Data[0]) (I guess.)
As said in your comment, it is the case, it is unaligned.
You can try two things.
First reserve memory with aligned_alloc which will give you an aligned buffer for TUN_RING.
Second, if TUN_RING is already aligned and the packet alignment is the problem, then you should give the correct offset to the head and tail:
recv_ring_->Head = 0; // <- aligned byte offset
recv_ring_->Tail = 0;
send_ring_->Head = 0;
send_ring_->Tail = 0;
Remember:
Head: Byte offset of the first packet in the ring. Its value must be a multiple of 4 and less than ring capacity.
Tail: Byte offset of the start of free space in the ring. Its value must be multiple of 4 and less than ring capacity.
The byte offset must be a multiple of 4.
You have to increment those skipped bytes to the buffer size. For that you may need to allocate extra space that won't be used, but I think it won't be too much.
In a second view to events, in the docs it says event has to be auto-reset:
Send.TailMoved: A handle to an auto-reset event created by the client that Wintun signals after it moves the Tail member of the send ring.
Receive.TailMoved: A handle to an auto-reset event created by the client that the client will signal when it changes Receive.Ring->Tail and Receive.Ring->Alertable is non-zero.
In your example, the event is auto-reset:
HANDLE send_event = ::CreateEvent(0, FALSE, FALSE, 0);
HANDLE recv_event = ::CreateEvent(0, FALSE, FALSE, 0);
but in the code you show (at top of question) isn't:
reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
I don't know if the parameter checking goes so far as to verify event auto-reset setting (not even if that's possible.) Moreover, the openvpn code creates them non auto-reset (although may be there is some code around to signal them before registering.)
Okay, after a lot of trial and error I have translated the whole setup routine from the WireGuard Go code (see here: https://github.com/WireGuard/wireguard-go ) to C++, which seems to make it work. It accepts the rings now just as in the first post and the device is shown as connected afterwards...
They are doing some registry tweaks after installing the device (see https://github.com/WireGuard/wireguard-go/blob/4369db522b3fd7adc28a2a82b89315a6f3edbcc4/tun/wintun/wintun_windows.go#L207 ) which I think takes the cake. Thanks for everyone helping in finding this.
For me the fix to get rid off ERROR_INVALID_PARAMETER (87) was to switch from x86 to x64 architecture in Visual Studio
The issue here is the alignment of the structs. You align your structs to 1 byte [#pragma pack(push, 1)] while the wintun driver does 8(/ZP8 in the solution). This will result in differing struct sizes and thus the size checks will fall through. Furthermore I would like to recommend that you use VirtualAlloc or the Mapping instead of malloc.
Now I get about 3.6GB data per second in memory, and I need to write them on my SSD continuously. I used CrystalDiskMark to test the writing speed of my SSD, it is almost 6GB per second, so I had thought this work should not be that hard.
![my SSD test result][1]:
[1]https://plus.google.com/u/0/photos/photo/106876803948041178149/6649598887699308850?authkey=CNbb5KjF8-jxJQ "test result":
My computer is Windows 10, using Visual Studio 2017 community.
I found this question and tried the highest voted answer. Unfortunately, the writing speed was only about 1s/GB for his option_2, far slower than tested by CrystalDiskMark. And then I tried memory mapping, this time writing becomes faster, about 630ms/GB, but still much slower. Then I tried multi-thread memory mapping, it seems that when the number of threads is 4, the speed was about 350ms/GB, and when I add the threads' number, the writing speed didn't go up anymore.
Code for memory mapping:
#include <fstream>
#include <chrono>
#include <vector>
#include <cstdint>
#include <numeric>
#include <random>
#include <algorithm>
#include <iostream>
#include <cassert>
#include <thread>
#include <windows.h>
#include <sstream>
// Generate random data
std::vector<int> GenerateData(std::size_t bytes) {
assert(bytes % sizeof(int) == 0);
std::vector<int> data(bytes / sizeof(int));
std::iota(data.begin(), data.end(), 0);
std::shuffle(data.begin(), data.end(), std::mt19937{ std::random_device{}() });
return data;
}
// Memory mapping
int map_write(int* data, int size, int id){
char* name = (char*)malloc(100);
sprintf_s(name, 100, "D:\\data_%d.bin",id);
HANDLE hFile = CreateFile(name, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);//
if (hFile == INVALID_HANDLE_VALUE){
return -1;
}
Sleep(0);
DWORD dwFileSize = size;
char* rname = (char*)malloc(100);
sprintf_s(rname, 100, "data_%d.bin", id);
HANDLE hFileMap = CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, dwFileSize, rname);//create file
if (hFileMap == NULL) {
CloseHandle(hFile);
return -2;
}
PVOID pvFile = MapViewOfFile(hFileMap, FILE_MAP_WRITE, 0, 0, 0);//Acquire the address of file on disk
if (pvFile == NULL) {
CloseHandle(hFileMap);
CloseHandle(hFile);
return -3;
}
PSTR pchAnsi = (PSTR)pvFile;
memcpy(pchAnsi, data, dwFileSize);//memery copy
UnmapViewOfFile(pvFile);
CloseHandle(hFileMap);
CloseHandle(hFile);
return 0;
}
// Multi-thread memory mapping
void Mem2SSD_write(int* data, int size){
int part = size / sizeof(int) / 4;
int index[4];
index[0] = 0;
index[1] = part;
index[2] = part * 2;
index[3] = part * 3;
std::thread ta(map_write, data + index[0], size / 4, 10);
std::thread tb(map_write, data + index[1], size / 4, 11);
std::thread tc(map_write, data + index[2], size / 4, 12);
std::thread td(map_write, data + index[3], size / 4, 13);
ta.join();
tb.join();
tc.join();
td.join();
}
//Test:
int main() {
const std::size_t kB = 1024;
const std::size_t MB = 1024 * kB;
const std::size_t GB = 1024 * MB;
for (int i = 0; i < 10; ++i) {
std::vector<int> data = GenerateData(1 * GB);
auto startTime = std::chrono::high_resolution_clock::now();
Mem2SSD_write(&data[0], 1 * GB);
auto endTime = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - startTime).count();
std::cout << "1G writing cost: " << duration << " ms" << std::endl;
}
system("pause");
return 0;
}
So I'd like to ask, is there any faster writing method for C++ to writing huge files? Or, why can't I write as fast as tested by CrystalDiskMark? How does CrystalDiskMark write?
Any help would be greatly appreciated. Thank you!
first of all this is not c++ question but os related question. for get maximum performance need need use os specific low level api call, which not exist in general c++ libs. from your code clear visible that you use windows api, so search solution for windows how minimum.
from CreateFileW function:
When FILE_FLAG_NO_BUFFERING is combined with FILE_FLAG_OVERLAPPED,
the flags give maximum asynchronous performance, because the I/O does
not rely on the synchronous operations of the memory manager.
so we need use combination of this 2 flags in call CreateFileW or FILE_NO_INTERMEDIATE_BUFFERING in call NtCreateFile
also extend file size and valid data length take some time, so better if final file at begin is known - just set file final size via NtSetInformationFile with FileEndOfFileInformation
or via SetFileInformationByHandle with FileEndOfFileInfo. and then set valid data length with SetFileValidData or via NtSetInformationFile with FileValidDataLengthInformation. set valid data length require SE_MANAGE_VOLUME_NAME privilege enabled when opening a file initially (but not when call SetFileValidData)
also look for file compression - if file compressed (it will be compressed by default if created in compressed folder) this is very slow writting. so need disbale file compression via FSCTL_SET_COMPRESSION
then when we use asynchronous I/O (fastest way) we not need create several dedicated threads. instead we need determine number of I/O requests run in concurrent. if you use CrystalDiskMark it actually run CdmResource\diskspd\diskspd64.exe for test and this is coresponded to it -o<count> parameter (run diskspd64.exe /? > h.txt for look parameters list).
use non Buffering I/O make task more hard, because exist 3 additional requirements:
Any ByteOffset passed to WriteFile must be a multiple of the sector
size.
The Length passed to WriteFile must be an integral of the sector
size
Buffers must be aligned in accordance with the alignment requirement
of the underlying device. To obtain this information, call
NtQueryInformationFile with FileAlignmentInformation
or GetFileInformationByHandleEx with FileAlignmentInfo
in most situations, page-aligned memory will also be sector-aligned,
because the case where the sector size is larger than the page size is
rare.
so almost always buffers allocated with VirtualAlloc function and multiple page size (4,096 bytes ) is ok. in concrete test for smaller code size i use this assumption
struct WriteTest
{
enum { opCompression, opWrite };
struct REQUEST : IO_STATUS_BLOCK
{
WriteTest* pTest;
ULONG opcode;
ULONG offset;
};
LONGLONG _TotalSize, _BytesLeft;
HANDLE _hFile;
ULONG64 _StartTime;
void* _pData;
REQUEST* _pRequests;
ULONG _BlockSize;
ULONG _ConcurrentRequestCount;
ULONG _dwThreadId;
LONG _dwRefCount;
WriteTest(ULONG BlockSize, ULONG ConcurrentRequestCount)
{
if (BlockSize & (BlockSize - 1))
{
__debugbreak();
}
_BlockSize = BlockSize, _ConcurrentRequestCount = ConcurrentRequestCount;
_dwRefCount = 1, _hFile = 0, _pRequests = 0, _pData = 0;
_dwThreadId = GetCurrentThreadId();
}
~WriteTest()
{
if (_pData)
{
VirtualFree(_pData, 0, MEM_RELEASE);
}
if (_pRequests)
{
delete [] _pRequests;
}
if (_hFile)
{
NtClose(_hFile);
}
PostThreadMessageW(_dwThreadId, WM_QUIT, 0, 0);
}
void Release()
{
if (!InterlockedDecrement(&_dwRefCount))
{
delete this;
}
}
void AddRef()
{
InterlockedIncrementNoFence(&_dwRefCount);
}
void StartWrite()
{
IO_STATUS_BLOCK iosb;
FILE_VALID_DATA_LENGTH_INFORMATION fvdl;
fvdl.ValidDataLength.QuadPart = _TotalSize;
NTSTATUS status;
if (0 > (status = NtSetInformationFile(_hFile, &iosb, &_TotalSize, sizeof(_TotalSize), FileEndOfFileInformation)) ||
0 > (status = NtSetInformationFile(_hFile, &iosb, &fvdl, sizeof(fvdl), FileValidDataLengthInformation)))
{
DbgPrint("FileValidDataLength=%x\n", status);
}
ULONG offset = 0;
ULONG dwNumberOfBytesTransfered = _BlockSize;
_BytesLeft = _TotalSize + dwNumberOfBytesTransfered;
ULONG ConcurrentRequestCount = _ConcurrentRequestCount;
REQUEST* irp = _pRequests;
_StartTime = GetTickCount64();
do
{
irp->opcode = opWrite;
irp->pTest = this;
irp->offset = offset;
offset += dwNumberOfBytesTransfered;
DoWrite(irp++);
} while (--ConcurrentRequestCount);
}
void FillBuffer(PULONGLONG pu, LONGLONG ByteOffset)
{
ULONG n = _BlockSize / sizeof(ULONGLONG);
do
{
*pu++ = ByteOffset, ByteOffset += sizeof(ULONGLONG);
} while (--n);
}
void DoWrite(REQUEST* irp)
{
LONG BlockSize = _BlockSize;
LONGLONG BytesLeft = InterlockedExchangeAddNoFence64(&_BytesLeft, -BlockSize) - BlockSize;
if (0 < BytesLeft)
{
LARGE_INTEGER ByteOffset;
ByteOffset.QuadPart = _TotalSize - BytesLeft;
PVOID Buffer = RtlOffsetToPointer(_pData, irp->offset);
FillBuffer((PULONGLONG)Buffer, ByteOffset.QuadPart);
AddRef();
NTSTATUS status = NtWriteFile(_hFile, 0, 0, irp, irp, Buffer, BlockSize, &ByteOffset, 0);
if (0 > status)
{
OnComplete(status, 0, irp);
}
}
else if (!BytesLeft)
{
// write end
ULONG64 time = GetTickCount64() - _StartTime;
WCHAR sz[64];
StrFormatByteSizeW((_TotalSize * 1000) / time, sz, RTL_NUMBER_OF(sz));
DbgPrint("end:%S\n", sz);
}
}
static VOID NTAPI _OnComplete(
_In_ NTSTATUS status,
_In_ ULONG_PTR dwNumberOfBytesTransfered,
_Inout_ PVOID Ctx
)
{
reinterpret_cast<REQUEST*>(Ctx)->pTest->OnComplete(status, dwNumberOfBytesTransfered, reinterpret_cast<REQUEST*>(Ctx));
}
VOID OnComplete(NTSTATUS status, ULONG_PTR dwNumberOfBytesTransfered, REQUEST* irp)
{
if (0 > status)
{
DbgPrint("OnComplete[%x]: %x\n", irp->opcode, status);
}
else
switch (irp->opcode)
{
default:
__debugbreak();
case opCompression:
StartWrite();
break;
case opWrite:
if (dwNumberOfBytesTransfered == _BlockSize)
{
DoWrite(irp);
}
else
{
DbgPrint(":%I64x != %x\n", dwNumberOfBytesTransfered, _BlockSize);
}
}
Release();
}
NTSTATUS Create(POBJECT_ATTRIBUTES poa, ULONGLONG size)
{
if (!(_pRequests = new REQUEST[_ConcurrentRequestCount]) ||
!(_pData = VirtualAlloc(0, _BlockSize * _ConcurrentRequestCount, MEM_COMMIT, PAGE_READWRITE)))
{
return STATUS_INSUFFICIENT_RESOURCES;
}
ULONGLONG sws = _BlockSize - 1;
LARGE_INTEGER as;
_TotalSize = as.QuadPart = (size + sws) & ~sws;
HANDLE hFile;
IO_STATUS_BLOCK iosb;
NTSTATUS status = NtCreateFile(&hFile,
DELETE|FILE_GENERIC_READ|FILE_GENERIC_WRITE&~FILE_APPEND_DATA,
poa, &iosb, &as, 0, 0, FILE_OVERWRITE_IF,
FILE_NON_DIRECTORY_FILE|FILE_NO_INTERMEDIATE_BUFFERING, 0, 0);
if (0 > status)
{
return status;
}
_hFile = hFile;
if (0 > (status = RtlSetIoCompletionCallback(hFile, _OnComplete, 0)))
{
return status;
}
static USHORT cmp = COMPRESSION_FORMAT_NONE;
REQUEST* irp = _pRequests;
irp->pTest = this;
irp->opcode = opCompression;
AddRef();
status = NtFsControlFile(hFile, 0, 0, irp, irp, FSCTL_SET_COMPRESSION, &cmp, sizeof(cmp), 0, 0);
if (0 > status)
{
OnComplete(status, 0, irp);
}
return status;
}
};
void WriteSpeed(POBJECT_ATTRIBUTES poa, ULONGLONG size, ULONG BlockSize, ULONG ConcurrentRequestCount)
{
BOOLEAN b;
NTSTATUS status = RtlAdjustPrivilege(SE_MANAGE_VOLUME_PRIVILEGE, TRUE, FALSE, &b);
if (0 <= status)
{
status = STATUS_INSUFFICIENT_RESOURCES;
if (WriteTest * pTest = new WriteTest(BlockSize, ConcurrentRequestCount))
{
status = pTest->Create(poa, size);
pTest->Release();
if (0 <= status)
{
MessageBoxW(0, 0, L"Test...", MB_OK|MB_ICONINFORMATION);
}
}
}
}
These are the suggestions that come to my mind:
stop all running processes that are using the disk, in particular
disable Windows Defender realtime protection (or other anti virus/malware)
disable pagefile
use Windows Resource Monitor to find processes reading or writing to your disk
make sure you write continuous sectors on disk
don't take into account file opening and closing times
do not use multithreading (your disk is using DMA so the CPU won't matter)
write data that is in RAM (obviously)
be sure to disable all debugging features when building (build a release)
if using M.2 PCIe disk (seems to be your case) make sure other PCIe
devices aren't stealing PCIe lanes to your disk (the CPU has a
limited number AND mobo too)
don't run the test from your IDE
disable Windows file indexing
Finally, you can find good hints on how to code fast writes in C/C++ in this question's thread: Writing a binary file in C++ very fast
One area that might give you improvement is to have your threads running constantly and each reading from a queue.
At the moment every time you go to write you spawn 4 threads (which is slow) and then they're deconstructed at the end of the function. You'll see a speedup of at least the cpu time of your function if you spawn the threads at the start and have them all reading from separate queue's in an infinite loop.
They'll simply check after a SMALL delay if there's anything in their queue, if their is they'll write it all. Your only issue then is making sure order of data is maintained.
We are building a quad copter robot for a competition, and one of the requirements is that we have to capture photos using the camera that is installed on the quad copter.
I wrote a simple OpenCV program that is able to capture them in .jpg format, but my camera or my program are unable to save "latitude" and "longitude" as EXIF on my images.
I've added to my robot a GPS module that can retrieve GPS data while capturing photos and save them to a text file.
So my main problem is adding these data from the text file to the pictures separately while they are captured.
I tried many libraries like:
easyexif
Exiv2
Phil Harvey
Also I tried them in:
Visual C++ .net 2015
Dev C++ (GCC)
Code Blocks (GCC)
and also I worked on PHP but I just can read and extract EIXFs but I can't write new data on my pictures.
How can I solve this problem?
you can use this code. I used Image::GetPropertyIdList method and Reading and Writing Metadata as a material for this code. unfortunately i didnt find any function in opencv that can manipulate Exif.
#include <windows.h>
#include <gdiplus.h>
#include <stdio.h>
#include <Math.h>
#pragma comment(lib,"gdiplus.lib")
using namespace Gdiplus;
int GetEncoderClsid(const WCHAR* format, CLSID* pClsid) //i copy this function from Doc.microsoft.com
{
UINT num = 0; // number of image encoders
UINT size = 0; // size of the image encoder array in bytes
ImageCodecInfo* pImageCodecInfo = NULL;
GetImageEncodersSize(&num, &size);
if (size == 0)
return -1; // Failure
pImageCodecInfo = (ImageCodecInfo*)(malloc(size));
if (pImageCodecInfo == NULL)
return -1; // Failure
GetImageEncoders(num, size, pImageCodecInfo);
for (UINT j = 0; j < num; ++j)
{
if (wcscmp(pImageCodecInfo[j].MimeType, format) == 0)
{
*pClsid = pImageCodecInfo[j].Clsid;
free(pImageCodecInfo);
return j; // Success
}
}
free(pImageCodecInfo);
return -1; }
int main(){
GdiplusStartupInput gdiplusStartupInput;
ULONG_PTR gdiplusToken;
GdiplusStartup(&gdiplusToken, &gdiplusStartupInput, NULL);
Status stat;
CLSID clsid;
Bitmap* bitmap = new Bitmap(L"Source.bmp"); // you can use any Image Format as Source
PropertyItem* propertyItem = new PropertyItem;
// Get the CLSID of the JPEG encoder.
GetEncoderClsid(L"image/jpeg", &clsid);
double dlan = 35.715298; // we supposed that your GPS data is Double if its not skip this step
// convert double to unsigned long array
double coord = dlan;
int sec = (int)round(coord * 3600);
int deg = sec / 3600;
sec = abs(sec % 3600);
int min = sec / 60;
sec %= 60;
unsigned long ulExifCoordFormatLan[6] = { deg, 1, min, 1, sec, 1 };
propertyItem->id = PropertyTagGpsLatitude;
propertyItem->length = sizeof(long) * 2 * 3;
propertyItem->type = PropertyTagTypeRational;
propertyItem->value = ulExifCoordFormatLan;
Status s = bitmap->SetPropertyItem(propertyItem);// saving image to the destination
stat = bitmap->Save(L"Dest.jpg", &clsid, NULL);
if (s == Ok && stat==Ok)
printf("Dest.jpg saved successfully .\n");
delete propertyItem;
delete bitmap;
GdiplusShutdown(gdiplusToken);
return 0;}
I'm currently working on PNG formats. And, I'm trying to understand its characteristics:
width, height, bytewidth, bytes per width, RGB pixel (red, green, blue)
Refering to this and this, width is the resolution width of the image in pixels, height is the resolution height of the image in pixels, bytes per width takes only 8 or 16 as value, RGB pixel's three items takes 0 to 255 as integer value.
But, I cannot understand what's a bytewidth?
Because when I try to convert one bmp image to a png one in C++ I get the following error:
Unhandled exception at 0x01038F6C in Project.exe: 0xC0000005: Access violation writing location 0x00BB9000.
I get this when I initialize the output png characteristics to:
width = 1600,
height = 900,
bytewidth = 1,
bytes per width = 1,
RGB pixel (red = 255, green = 255, blue = 255)
using this code:
#define WIN32_LEAN_AND_MEAN
#define _CRT_SECURE_NO_DEPRECATE
#include <png.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
typedef struct _RGBPixel {
uint8_t blue;
uint8_t green;
uint8_t red;
} RGBPixel;
/* Structure for containing decompressed bitmaps. */
typedef struct _RGBBitmap {
RGBPixel *pixels;
size_t width;
size_t height;
size_t bytewidth;
uint8_t bytes_per_pixel;
} RGBBitmap;
/* Returns pixel of bitmap at given point. */
#define RGBPixelAtPoint(image, x, y) \
*(((image)->pixels) + (((image)->bytewidth * (y)) \
+ ((x) * (image)->bytes_per_pixel)))
/* Attempts to save PNG to file; returns 0 on success, non-zero on error. */
int save_png_to_file(RGBBitmap *bitmap, const char *path)
{
FILE *fp = fopen(path, "wb");
png_structp png_ptr = NULL;
png_infop info_ptr = NULL;
size_t x, y;
png_uint_32 bytes_per_row;
png_byte **row_pointers = NULL;
if (fp == NULL) return -1;
/* Initialize the write struct. */
png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
if (png_ptr == NULL) {
fclose(fp);
return -1;
}
/* Initialize the info struct. */
info_ptr = png_create_info_struct(png_ptr);
if (info_ptr == NULL) {
png_destroy_write_struct(&png_ptr, NULL);
fclose(fp);
return -1;
}
/* Set up error handling. */
if (setjmp(png_jmpbuf(png_ptr))) {
png_destroy_write_struct(&png_ptr, &info_ptr);
fclose(fp);
return -1;
}
/* Set image attributes. */
png_set_IHDR(png_ptr,
info_ptr,
bitmap->width,
bitmap->height,
8,
PNG_COLOR_TYPE_RGB,
PNG_INTERLACE_NONE,
PNG_COMPRESSION_TYPE_DEFAULT,
PNG_FILTER_TYPE_DEFAULT);
/* Initialize rows of PNG. */
bytes_per_row = bitmap->width * bitmap->bytes_per_pixel;
row_pointers = (png_byte **)png_malloc(png_ptr, bitmap->height * sizeof(png_byte *));
for (y = 0; y < bitmap->height; ++y) {
uint8_t *row = (uint8_t *)png_malloc(png_ptr, sizeof(uint8_t)* bitmap->bytes_per_pixel);
row_pointers[y] = (png_byte *)row;
for (x = 0; x < bitmap->width; ++x) {
RGBPixel color = RGBPixelAtPoint(bitmap, x, y);
*row++ = color.red;
*row++ = color.green; /************* MARKED LINE ***************/
*row++ = color.blue;
}
}
/* Actually write the image data. */
png_init_io(png_ptr, fp);
png_set_rows(png_ptr, info_ptr, row_pointers);
png_write_png(png_ptr, info_ptr, PNG_TRANSFORM_IDENTITY, NULL);
/* Cleanup. */
for (y = 0; y < bitmap->height; y++) {
png_free(png_ptr, row_pointers[y]);
}
png_free(png_ptr, row_pointers);
/* Finish writing. */
png_destroy_write_struct(&png_ptr, &info_ptr);
fclose(fp);
return 0;
}
int main()
{
RGBBitmap rgbbitmap;
rgbbitmap.height = 1600;
rgbbitmap.width = 900;
rgbbitmap.bytes_per_pixel = 1;
rgbbitmap.bytewidth = 1;
RGBPixel rgbpixel;
rgbpixel.blue = 255;
rgbpixel.green = 255;
rgbpixel.red = 255;
rgbbitmap.pixels = &rgbpixel;
save_png_to_file(&rgbbitmap, "abc.bmp");
return 0;
}
I'm actually getting that error on the marked line.
Any brilliant suggestion, please?
After setting
rgbbitmap.bytes_per_pixel = 3; // was 1 which is probably wrong
Surely
uint8_t *row = (uint8_t *)png_malloc(png_ptr, sizeof(uint8_t)* bitmap->bytes_per_pixel);
must be
uint8_t *row = (uint8_t *)png_malloc(png_ptr, sizeof(uint8_t)* bitmap->bytes_per_pixel*bitmap->width);
, i.e. each row must hold enough space for all bytes in that row ;-), which is 3 bytes per pixel times pixels in a row. You actually compute that value, bytes_per_row, but for some reason fail to use it.
This
uint8_t *row = (uint8_t *)png_malloc(png_ptr, sizeof(uint8_t)* bitmap->bytes_per_pixel);
looks suspicious - I don't know how png_malloc() works, but I guess you're allocating sizeof(uint8_t) * BytesPerPixel (which is probably sizeof(uint8_t) * 4) instead of sizeof(uint8_t) * NumberOfPixels (which should be considerably larger).
I think your rgbbitmap.bytes_per_pixel = 1 is wrong.
If your work in RGB8 your bytes_per_pixel = 3, in RGBA8 is 4 and so on.