I have the following c++ code which writes "Line from #" to a file while managing a file lock. I am running this code on two different computers, which share at least some of their memory. That is I can access my files by logging onto either of these computers.
On the first computer I run the program as ./test 1 (e.g. so it will print Line from 1 20,000 times) and on the second computer I run the program as ./test 17. I am starting these programs close enough in time so that the writes to file.txt should be interleaved and controlled by the file locks.
The problem is that I am losing output as the file has 22,770 newlines, but it should have exactly 40,000 newlines.
wc file.txt
22770 68310 276008 file.txt
Also,
cat -n file.txt | grep 18667
18667 ne from 17
My question is why are my file locks not preventing file overwriting, and how can I fix my code so that multiple processes can write to the same file without file loss.
#include <unistd.h>
#include <fcntl.h>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <sstream>
#include <iostream>
using namespace std;
void inline Set_Lck(struct flock &flck, const int fd)
{
flck.l_type = F_WRLCK;
if (fcntl(fd, F_SETLKW, &flck) == -1) {
perror("fcntl");
exit(1);
}
}
void inline Release_Lck(struct flock &flck, const int fd)
{
flck.l_type = F_UNLCK;
if (fcntl(fd,F_SETLK,&flck) == -1) {
perror("fcntl");
exit(1);
}
}
void Print_Spec(fstream &fout, ostringstream &oss,struct flock &flck, const int fd)
{
Set_Lck(flck,fd);
fout.seekp(0,ios_base::end);
fout << oss.str() << endl;
flush(fout);
Release_Lck(flck,fd);
}
int main(int argc, char **argv)
{
int fd_cd;
struct flock flock_cd;
ostringstream oss;
fstream comp_data;
const string s_cd_lck = "file_lock.txt";
const string s_cd = "file.txt";
int my_id;
if (argc == 1) {
my_id = 0;
} else if (argc == 2) {
my_id = atoi(argv[1]);
} else {
fprintf(stderr,"error -- usage ./test [my_id]\n");
exit(1);
}
/* Open file computed_data.txt for writing; create it if non-existent.*/
comp_data.open(s_cd.c_str(),ios::app|ios::out);
if (comp_data.fail()) {
perror("comp_data.open");
exit(1);
}
/* Open file that we will be locking. */
fd_cd = open(s_cd_lck.c_str(),O_CREAT|O_WRONLY,0777);
if (fd_cd == -1) {
perror("fd_cd = open");
exit(1);
}
/* Set up the lock. */
flock_cd.l_type = F_WRLCK;
flock_cd.l_whence = SEEK_SET;
flock_cd.l_start = 0;
flock_cd.l_len = 0;
flock_cd.l_pid = getpid();
for (int i = 0; i < 20000; ++i) {
oss.str(""); /* Yes, this can be moved outside the loop. */
oss << "Line from " << my_id << endl;
Print_Spec(comp_data,oss,flock_cd,fd_cd);
}
return 0;
}
I am using c++ and this program is running on Red Hat Enterprise Linux Server release 7.2 (Maipo).
My Research
I am not sure if part of the answer comes from the following Stackoverflow post (https://stackoverflow.com/a/2059059/6417898) where they state that "locks are bound to processes."
At this website (http://perl.plover.com/yak/flock/samples/slide005.html), the author dissuades against using LOCK_UN with flock and suggests closing the file each time and reopening it as needed, so as to flush the file buffer. I don't know if this carries over with fcntl or if this is even necessary if flush the file buffer manually.
Related
It's a real-time capture system, I need to get the latest changes from a file which is occasionally edited(mostly add content) by other applications.
In other words, how can I get content that added in the period when I open it without reopening the file?
#include <iostream>
#include <fstream>
#include <string>
using namespace std;
int main(){
ifstream tfile("temp.txt",ios::in);
if(!tfile){
cout<<"open failed"<<endl;
return 0;
}
string str;
while(1){
if(tfile.eof())
continue;
getline(tfile,str);
cout<<str<<endl;
}
tfile.close();
}
C++ / C Solution
If you are looking for a c++ solution you can use the following functions that I had created a while back:
#include <iostream>
#include <string>
// For sleep function
#ifdef _WIN32
#include <Windows.h>
#else
#include <unistd.h>
#endif
using namespace std;
void watchLogs(const char *FILENAME) {
FILE * f;
unsigned size = 0;
f = fopen(FILENAME , "r");
char c;
while (true) {
if (!size) { // will print content of your log file. If you just want the updates you can remove the current content except the first two lines;
fseek(f, 0, SEEK_END);
size =(unsigned long)ftell(f) ;
fseek (f, 0, SEEK_SET);
char buffer[size + 1];
fread ( buffer, 1, size, f );
buffer[size] = '\0';
cout << buffer << "\n";
}
else if ((c = (char)fgetc(f)) >= 0) {
fseek(f, 0, SEEK_END); // reach end of file
int BUFFER_SIZE =(unsigned long)ftell(f) - size; // save the length of the update to your logs
char buffer[BUFFER_SIZE + 1]; // prepare a buffer to print the characters
fseek(f,-BUFFER_SIZE,SEEK_END); // rewind BUFFER_SIZE characters before the EOF
int i = 0;
do {buffer[i++] = (char)fgetc(f);} while(i < BUFFER_SIZE); // copy to buffer
buffer[i] = '\0'; // don't forget to NULL terminate your buffer
cout << buffer << "\n";
size += i; // increment the size of the current file
}
}
sleep(3); // updates are checked every 3 seconds to avoid running the cpu at fullspeed, you could set the new logs to show up every minutes or every seconds, up to you.
fclose(f);
}
And you can test it with:
int main(int argc, char **argv) {
if (argc < 2)
return 1;
const char *FILENAME = argv[1];
watchLogs(FILENAME);
return 0;
}
./a.out mysql_binary.log
I could have used stringstreamer but I like that this version would also work with c files with some minor tweaks (can't use string).
I hope you will find it helpful!
NB: This assume that your file will only grow and that the changes will be appended to the end of your file.
NB2: This program is not segfault proof, you may want to check the return of fopen etc
Inotify
If you use Linux you could also potentially go for inotify:
Download inotify: sudo apt-get install -y inotify-tools
Then create the following script mywatch.sh
while inotifywait -e close_write $1; do ./$1; done
Give permission to execute:
add chmox +x mywatch.sh
and call it with ./watchit.sh mysql_binary.log
Is there a simple way to get the number of files opened by a c++ program.
I would like to do it from my code, ideally in C++.
I found this blog article which is using a loop through all the available file descriptor and testing the result of fstat but I am wondering if there is any simpler way to do that.
Edit
It seems that there are no other solution than keeping a count of the files opened. Thanks to everybody for your help.
Kevin
Since the files are FILE *, we could do something like this:
In a headerfile that gets included everywhere:
#define fopen(x, y) debug_fopen(x, y, __FILE__, __LINE__)
#define fclose(x) debug_fclose(x)
in "debugfile.cpp" (must obviously NOT use the above #define's)
struct FileInfo
{
FileInfo(const char *nm, const char fl, int ln) :
name(nm), file(fl), line(ln) {}
std::string name;
const char *file;
int line;
};
std::map<FILE*, FileInfo> filemap;
FILE *debug_fopen(const char *fname, const char *mode, const char *file, int line)
{
FILE *f = fopen(fname, mode);
if (f)
{
FileInfo inf(fname, file, line);
filemap[f] = inf;
}
}
int debug_fclose(FILE *f)
{
int res = fclose(f);
filemap.erase(f);
return res;
}
// Called at some points.
void debug_list_openfiles()
{
for( i : filemap )
{
cerr << "File" << (void *) i.first << " opened as " << i.second.name
<< " at " << i.second.file << ":" << i.second.line << endl;
}
}
(I haven't compiled this code, and it's meant to show the concept, it may have minor bugs, but I think the concept would hold - as long as your code, and not some third party library is leaking)
This is a legitimate question: I count open file descriptors in unit tests to verify none has leaked. On Linux systems there's one entry in /proc/self/fd for each open file descriptor, so you just have to count them. In c++17 it looks like this:
long file_descriptor_count() {
return std::distance(std::filesystem::directory_iterator("/proc/self/fd"), std::filesystem::iterator{});
}
there is a good practice that the scope of file opened in the smallest possible, open dump all information you want, or buffer into the fd, then close.
so this mean usual case we will have 3 fd the std in/out/err, plus all opened files.
keep track of your open files manually is the best, if you keep files opened.
put a global fdCounter variable,
increment it after a successful file open, decremented after closing
If you are under linux, this information is available under /proc/you_pid/fd.
Then use, lstat on each file descriptor to keep only regular files.
If you encapsulated it properly, it should be simple to add reference counters or logging to it and print them to the console.
One approach to debug it is to override the open calls with your own implementation and from there call the real thing. Then you can also put some logging in, to see if you loose file descriptors. How do you open the files? With open() or are you using fopen()?
Something like this maybe:
#include <fstream>
#include <iostream>
#include <stdlib.h>
#include <fcntl.h>
inline int log_open(char *p, int flags, int mode)
{
int fd = ::open(p, flags, mode);
std::cout << "OPEN: " << fd << std::endl;
return fd;
}
inline int log_close(int fd)
{
int rc = ::close(fd);
std::cout << "CLOSE: " << fd << std::endl;
return rc;
}
#define open(p, f, m) log_open(p, f, m)
#define close(fd) log_close(fd)
int main(int argc, char *argv[])
{
int fd = open("tmp.txt", O_RDWR | O_CREAT | O_TRUNC, 0666);
std::cout << "FD: " << fd << std::endl;
if(fd != -1)
close(fd);
return 0;
}
In my experience, by the time you need to count the number of file descriptors, you don't know where they were opened, by what submodule or library. Thus, wrapping open/close is not a viable strategy. Brute-force counting seems to be the only way.
The domain with the orig blog post no longer resolves in DNS. I copy two proposals from Find current number of open filehandle ( NOT lsof )
int j, n = 0;
// count open file descriptors
for (j = 0; j < FDMAX; ++j) // FDMAX should be retrieved from process limits,
// but a constant value of >=4K should be
// adequate for most systems
{
int fd = dup (j);
if (fd < 0)
continue;
++n;
close (fd);
}
printf ("%d file descriptors open\n", n);
and also this:
#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>
int main (void)
{
DIR *dp;
struct dirent *ep;
dp = opendir ("/proc/MYPID/fd/");
if (dp != NULL)
{
while (ep = readdir (dp))
puts (ep->d_name);
(void) closedir (dp);
}
else
perror ("Couldn't open the directory");
return 0;
}
I have create following test case for simulating the issue.I have compiled the source code and able to simulate the issue.
1)When the system command,we got some console out ( i.e your job submitted) which is redirect to file using dup2 and create file .stdout.
When I try to read this file as I need job submission information and I did not get data which was on console out. I was able to get data which I wrote it.( confirm file operation).
Can we not read console output from the file which is create by the child process.
*change rundir and cmd
#include <string>
#include <vector>
#include <utility>
#include <sstream>
#include <fstream>
#include <iostream>
#include <iterator>
#include <algorithm>
#include <functional>
#include <map>
#include <sys/stat.h>
#include <fcntl.h>
using namespace std;
int child();
int main()
{
string rundir = "/temp";
pid_t id =child();
string _xafile;
string _afile;
string _afilecmd;
string line;
stringstream ss(stringstream::in | stringstream::out);
ss<<int(id);
_xafile = rundir;
_xafile = _xafile + "/" + ss.str()+".stdout";
cout<<" _xafile is "<<_xafile<<endl;
string cmd ="cat "+ _xafile + ">" + rundir + "/" + "process.txt";
_afile = rundir;
_afile = _afile + "/" + "process.txt";
_afilecmd = "rm -rf "+ _afile;
system(cmd.c_str());
ifstream xafile(_afile.c_str());
while(xafile)
{
string word;
xafile >> word;
cout<<word<<" ";
/* try to read console output but did not read data */
}
system(_afilecmd.c_str());
return 0;
}
int child()
{
string rundir = "/tmp";
string cmd = " tool <input file> -o <outputfile>";
const char* std_out_file;
pid_t pid = fork();
if (pid < 0) {
return -1;
}
if (pid == 0) {
pid_t mypid = getpid();
stringstream sm;
sm << rundir << "/";
if (strlen(std_out_file) == 0)
sm << mypid << ".stdout";
else
sm << std_out_file;
int fd = open(sm.str().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
dup2(fd, 1);
dup2(fd, 2);
int fd2 = open("/dev/zero", O_RDONLY);
dup2(fd2, 0);
cout<<cmd <<endl<<endl;
// execl("/bin/sh", "/bin/sh", "-c", cmd, NULL);
system(cmd.c_str());
/*When system call it generate console output like : your job submitted */
/* redirect to file */
exit(-1);
}
if (pid > 0)
return pid;
cout<<" child is done"<<endl;
return 0;
}
It's not entirely clear what your thinking is here - your code appears to fork a child which does a bunch of fancy io stuff to try and redirect your applications stdout to a file, and then runs a command with system(). The command in your code has it's own redirects, specifically via "-o" so it probably isn't writing to stdout.
Back in the parent process, you try to open for reading the same file your child process will open for writing. You've got no synchronization, so they could happen in any order. You appear to be trying to use "cat" to read the file and trying to read the stdout of the cat?
What I think you are actually trying to do is the C/C++ equivalent of Perl's
$foo = `ls -l /tmp`;
or something similar - to execute a command and capture the output.
A better way to do this would be to use pipes.
#include <iostream>
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
enum { WritePipe = 1, ReadPipe = 0 };
int main(int argc, const char** argv)
{
int pipes[2]; // each process is going to have its own file descriptor.
if(pipe(pipes) != 0) {
perror("pipe failed");
return -1;
}
pid_t pid = fork();
if(pid == 0) {
// child
close(pipes[ReadPipe]); // close the parent pipe in our context.
// redirect stdout and stderr to the pipe.
dup2(pipes[WritePipe], STDOUT_FILENO);
dup2(pipes[WritePipe], STDERR_FILENO);
// close(STDERR_FILENO); // <- or just do this to close stderr.
int result = system("ls -ltr /etc");
close(pipes[WritePipe]);
return result;
}
if(pid < 0) {
perror("fork failed");
return -1;
}
// Parent process has launched the child.
close(pipes[WritePipe]);
char buffer[4097];
int bytesRead;
while((bytesRead = read(pipes[ReadPipe], buffer, sizeof(buffer) - 1)) > 0) {
buffer[bytesRead] = 0;
std::cout << buffer;
}
std::cout << std::endl;
close(pipes[ReadPipe]);
return 0;
}
I'm calling a LINUX command from within a C++ programme which creates the following output. I need to copy the first column of the output to a C++ variable (say a long int). How can I do it?? If that is not possible how can I copy this result into a .txt file with which I can work with?
Edit
0 +0
2361294848 +2361294848
2411626496 +50331648
2545844224 +134217728
2713616384 +167772160
I have this stored as a file, file.txt and I'm using the following code to
extract the left column with out the 0 to store it at integers
string stringy="";
int can_can=0;
for(i=begin;i<length;i++)
{
if (buffer[i]==' ' && can_can ==1) //**buffer** is the whole text file read in char*
{
num=atoi(stringy.c_str());
array[univ]=num; // This where I store the values.
univ+=1;
can_can=1;
}
else if (buffer[i]==' ' && can_can ==0)
{
stringy="";
}
else if (buffer[i]=='+')
{can_can=0;}
else{stringy.append(buffer[i]);}
}
I'm getting a segmentation error for this. What can be done ?
Thanks in advance.
Just create a simple streambuf wrapper around popen()
#include <iostream>
#include <stdio.h>
struct SimpleBuffer: public std::streambuf
{
typedef std::streambuf::traits_type traits;
typedef traits::int_type int_type;
SimpleBuffer(std::string const& command)
: stream(popen(command.c_str(), "r"))
{
this->setg(&c[0], &c[0], &c[0]);
this->setp(0, 0);
}
~SimpleBuffer()
{
if (stream != NULL)
{
fclose(stream);
}
}
virtual int_type underflow()
{
std::size_t size = fread(c, 1, 100, stream);
this->setg(&c[0], &c[0], &c[size]);
return size == 0 ? EOF : *c;
}
private:
FILE* stream;
char c[100];
};
Usage:
int main()
{
SimpleBuffer buffer("echo 55 hi there Loki");
std::istream command(&buffer);
int value;
command >> value;
std::string line;
std::getline(command, line);
std::cout << "Got int(" << value << ") String (" << line << ")\n";
}
Result:
> ./a.out
Got int(55) String ( hi there Loki)
It is popen you're probably looking for. Try
man popen
.
Or see this little example:
#include <iostream>
#include <stdio.h>
using namespace std;
int main()
{
FILE *in;
char buff[512];
if(!(in = popen("my_script_from_command_line", "r"))){
return 1;
}
while(fgets(buff, sizeof(buff), in)!=NULL){
cout << buff; // here you have each line
// of the output of your script in buff
}
pclose(in);
return 0;
}
Unfortunately, it’s not easy since the platform API is written for C. The following is a simple working example:
#include <cstdio>
#include <iostream>
int main() {
char const* command = "ls -l";
FILE* fpipe = popen(command, "r");
if (not fpipe) {
std::cerr << "Unable to execute commmand\n";
return EXIT_FAILURE;
}
char buffer[256];
while (std::fgets(buffer, sizeof buffer, fpipe)) {
std::cout << buffer;
}
pclose(fpipe);
}
However, I’d suggest wrapping the FILE* handle in a RAII class to take care of resource management.
You probably want to use popen to execute the command. This will give you a FILE * that you can read its output from. From there, you can parse out the first number with (for example) something like:
fscanf(inpipe, "%d %*d", &first_num);
which, just like when reading from a file, you'll normally repeat until you receive an end of file indication, such as:
long total = 0;
while (1 == fscanf(inpipe, "%l %*d", &first_num))
total = first_num;
printf("%l\n", total);
I am working in Ubuntu. I want to monitor a folder and print every event that pops up in the subfolders (print files).
I have the following code but it doesn't work. When executed, there is no println of the events.
In the second code I only see the events from the folder. The events from each subfolder do not pop up.
#include <string>
#include <iostream>
#include <stdio.h>
using namespace std;
std::string exec(char* cmd) {
FILE* pipe = popen(cmd, "r");
if (!pipe) return "ERROR";
char buffer[256];
std::string result = "";
while(!feof(pipe)) {
if(fgets(buffer, 256, pipe) != NULL)
result += buffer;
}
pclose(pipe);
cout<<"result is: "<<result<<endl;
return result;
}
int main()
{
//while(1)
//{
string s=exec((char*)"inotifywait -rme create /home/folder/");
cout << s << endl;
//}
return 0;
}
This code only prints the events from the folder I'm monitoring. It doesn't print the events from each subfolder. I don't know how to improve it for my needs.
#include <sys/inotify.h>
#include <sys/ioctl.h>
#include <iostream>
void processNewFiles(int fd, int wd);
int main(int argc, char** argv)
{
const char* dirPath = "/home/folder/" ;//argv[1];
int fd = inotify_init();
int wd = inotify_add_watch(fd, dirPath, IN_CREATE);
if (wd)
{
processNewFiles(fd, wd);
inotify_rm_watch(fd, wd);
}
}
void processNewFiles(int fd, int wd)
{
bool done = false;
do
{
int qLen = 0;
ioctl(fd, FIONREAD, &qLen);
char* buf = new char[qLen];
int num = read(fd, buf, qLen);
if (num == qLen)
{
inotify_event* iev = reinterpret_cast<inotify_event*>(buf);
if (iev->wd == wd && iev->mask & IN_CREATE)
{
std::cout << "New file created: " << iev->name << std::endl;
}
}
delete [] buf;
} while (!done);
}
Your second solution does not work because inotify_add_watch is not working recursivly. You would have to add watches for subdirectories manually. As this might be annoying, it is also possible to use the utility inotifywait as you do in your first example.
Your first example is not working because you're reading from the pipe forever. If you kill the inotifywait process (e.g. if you're the only person on the machine and this is the only inotifywait process just using "killall inotifywait") you will get your output because you'll break out of the loop reading from the pipe. If you output something inside the loop, it will work, too.