Is there anything in python that can replicate the functionality of freopen() in C or C++? To be precise, I want to replicate the functionality of:
freopen("input.txt","r",stdin);
and
freopen("output.txt","w",stdout);
And then use the same (standard) functions for console I/O for file I/O. Any ideas?
sys.stdout is simply file object, so, you can reopen it to another destination
out = sys.stdout
sys.stdout = open('output.txt', 'w')
// do some work
sys.stdout = out
out is only for recovering sys.stdout destination to default after work (as suggested Martijn Pieters - you can recover it by using sys.__stdout__, or not recover at all, if you don't need it).
Try this:
import sys
sys.stdin = open('input.txt', 'r')
sys.stdout = open('output.txt', 'w')
Text files are self explanatory.
You can now run this code on Sublime Text or any other text editor.
If you're working on *nix platform, you can write your own freopen.
def freopen(f,option,stream):
import os
oldf = open(f,option)
oldfd = oldf.fileno()
newfd = stream.fileno()
os.close(newfd)
os.dup2(oldfd, newfd)
import sys
freopen("hello","w",sys.stdout)
print "world"
You may also want to look at the contextmanager decorator in contextlib for temporary redirection:
from contextlib import contextmanager
import sys
#contextmanager
def stdout_redirected(new_stdout):
save_stdout = sys.stdout
sys.stdout = new_stdout
try:
yield
finally:
sys.stdout = save_stdout
Example:
with open(filename, "w") as f:
with stdout_redirected(f):
print "Hello"
This should help:
import sys
def freopen(filename, mode):
if mode == "r":
sys.stdin = open(filename, mode)
elif mode == "w":
sys.stdout = open(filename, mode)
# ---- MAIN ----
freopen("input.txt", "r")
freopen("output.txt", "w")
Related
I'm trying to make a recursive function by using os.listdir(), and I am having troble looping to all my directories and list out all the files and directories.
I know it's better using os.tree() for solving this kind of problem, but i want to see how to solve this by using os.listdir().
Here are my current code:
#!/bin/usr/py
from os.path import abspath
from os.path import isfile, isdir
import os
import sys
dir = sys.argv[1]
def recursive(dir):
files = os.listdir(dir)
for obj in files:
if isfile(obj):
print obj
elif isdir(obj):
print obj
recursive(abspath(obj))
#no idea why this won't work???
recursive(dir)
Your issue comes from abspath(obj), try replacing it by os.path.join(dir, obj) to have real path to your obj (I tested it on my env)
Thanks Gabriel and Emilrn ! this was exactly what I was looking for to recursively get the list of files from a parent directory provided for one of my projects. Just leaving the updated code here for someone who needs it later.
#!/bin/usr/py
import os
import sys
dir = sys.argv[1]
def recursive(dir):
files = os.listdir(dir)
for obj in files:
if os.path.isfile(os.path.join(dir,obj)):
print ("File : "+os.path.join(dir,obj))
elif os.path.isdir(os.path.join(dir,obj)):
recursive(os.path.join(dir, obj))
else:
print ('Not a directory or file %s' % (os.path.join(dir, obj))
recursive(dir)
I have a function that first examines whether a txt file exists and if it does not it creates one. If the txt file already exists it reads the info. I am trying to write unittests to examine whether the logic of the function is correct. I want to patch things like existence of files, creation of files and reading of files.
The function to be tested looks like this:
import json
import os.path
def read_create_file():
filename = 'directory/filename.txt'
info_from_file = []
if os.path.exists(filename):
with open(filename, 'r') as f:
content = f.readlines()
for i in range(len(content)):
info_from_file.append(json.loads(content[i]))
return info_from_file
else:
with open(filename, 'w') as f:
pass
return []
The unittest looks like this:
import unittest
import mock
from mock import patch
class TestReadCreateFile(unittest.TestCase):
def setUp(self):
pass
def function(self):
return read_create_file()
#patch("os.path.exists", return_value=False)
#mock.patch('directory/filename.txt.open', new=mock.mock_open())
def test_file_does_not_exist(self, mock_existence, mock_open_patch):
result = self.function()
self.assertEqual(result, (True, []))
ERROR: ImportError: Import by filename is not supported.
or like this:
import unittest
import mock
from mock import patch
#patch("os.path.exists", return_value=False)
def test_file_not_exist_yet(self, mock_existence):
m = mock.mock_open()
with patch('__main__.open', m, create=True):
handle = open('directory/filename.txt', 'w')
result = self.function()
self.assertEqual(result, (True, {}))
ERROR:
IOError: [Errno 2] No such file or directory: 'directory/filename.txt'
As a newbie I cannot seem to get my head around a solution, any help is greatly appreciated.
Thank you
You're mocking os.path.exists wrong. When you patch you patch from the file under test.
#patch("path_to_method_under_test.path.exists", return_value=False)
def test_file_not_exist_yet(self, mock_existence):
I am new to Python and as my first project I am attempting to convert a Python2 script to Python3.
The script is failing when it attempts to serialize a class using pickle.
It seems as though it is failing as I am trying to save a class which uses the Cmd CLI.
This code works using Python2.
Can anyone tell me what is wrong with the script and how I fix it?
import sys
import cmd
try:
import pickle as pickle
except:
import pickle
import os.path
def main():
app = Labyrinth()
turnfile = "turn0.lwot"
app.Save(turnfile)
class CLI(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
class Labyrinth(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
def Save(self, fname):
with open(fname, 'wb') as f:
pickle.dump(self,f, 2)
f.close()
print ("Save Successful!")
sys.exit()
if __name__ == '__main__':
main()
Not all objects are picklable. In particular, file objects are problematic because you can't generally restore their state later. cmd.Cmd holds stdin and stdout file objects and that should make them unpicklable. I was quite surprised that it worked in python 2, but it didn't really... Even though the stdin and stdout pickled, the unpickled object you get back later doesn't work, as in this example:
>>> import sys
>>> import pickle
>>> sys.stdout.write('foo\n')
foo
>>> serialized = pickle.dumps(sys.stdout, 2)
>>> stdout = pickle.loads(serialized)
>>> stdout.write('bar\n')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: I/O operation on closed file
>>>
So, even though this bit of code didn't fail, the object shouldn't be usable later. You can add a few special methods to an object that let you fix objects so they can be serialized. Here, I've stripped the bad attributes on save and added them back on restore. Now you can pickle, unpickle and it actually works when you are done.
import sys
import cmd
try:
import cPickle as pickle
except:
import pickle
import os.path
def main():
app = Labyrinth()
turnfile = "turn0.lwot"
app.Save(turnfile)
class CLI(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
class Labyrinth(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
def Save(self, fname):
with open(fname, 'wb') as f:
pickle.dump(self,f, pickle.HIGHEST_PROTOCOL)
f.close()
print ("Save Successful!")
sys.exit()
def __getstate__(self):
# stdin/out are unpicklable. We'll get new ones on load
return tuple(((k,v) for k,v in self.__dict__.items()
if k not in ('stdin', 'stdout')))
def __setstate__(self, state):
self.__dict__.update(state)
self.stdin = sys.stdin
self.stdout = sys.stdout
if __name__ == '__main__':
main()
Playing with the protocol doesn't help. The full error message (which you should have included) is:
1027:~/mypy$ python3 stack41334887.py
Traceback (most recent call last):
File "stack41334887.py", line 33, in <module>
main()
File "stack41334887.py", line 14, in main
app.Save(turnfile)
File "stack41334887.py", line 27, in Save
pickle.dump(self,f, 3, fix_imports=True)
TypeError: cannot serialize '_io.TextIOWrapper' object
Python3 made some major changes in the io system. This TextIOWrapper is, I think new to Py3.
https://docs.python.org/3.1/library/io.html#io.TextIOWrapper
Can I use multiprocessing.Pool in a method of a class? also had problems serializing a TextIOWrapper.
=========
So inspireed by #tdelaney, I checked the stdin for my PY3 session:
In [1212]: sys.stdin
Out[1212]: <_io.TextIOWrapper name='<stdin>' mode='r' encoding='UTF-8'>
So that's the thing that can't be serialized.
Is there any way in Pandas to capture the warning produced by setting error_bad_lines = False and warn_bad_lines = True? For instance the following script:
import pandas as pd
from StringIO import StringIO
data = StringIO("""a,b,c
1,2,3
4,5,6
6,7,8,9
1,2,5
3,4,5""")
pd.read_csv(data, warn_bad_lines=True, error_bad_lines=False)
produces the warning:
Skipping line 4: expected 3 fields, saw 4
I'd like to store this output to a string so that I can eventually write it to a log file to keep track of records that are being skipped.
I tried using the warning module but it doesn't appear as though this "warning" is of the traditional sense. I'm using Python 2.7 and Pandas 0.16.
I think it isn't implemented to pandas.
source1, source2
My solutions:
1. Pre or after processing
import pandas as pd
import csv
df = pd.read_csv('data.csv', warn_bad_lines=True, error_bad_lines=False)
#compare length of rows by recommended value:
RECOMMENDED = 3
with open('data.csv') as csv_file:
reader = csv.reader(csv_file, delimiter=',')
for row in reader:
if (len(row) != RECOMMENDED):
print ("Length of row is: %r" % len(row) )
print row
#compare length of rows by length of columns in df
lencols = len(df.columns)
print lencols
with open('data.csv') as csv_file:
reader = csv.reader(csv_file, delimiter=',')
for row in reader:
if (len(row) != lencols):
print ("Length of row is: %r" % len(row) )
print row
2. Replaces sys.stdout
import pandas as pd
import os
import sys
class RedirectStdStreams(object):
def __init__(self, stdout=None, stderr=None):
self._stdout = stdout or sys.stdout
self._stderr = stderr or sys.stderr
def __enter__(self):
self.old_stdout, self.old_stderr = sys.stdout, sys.stderr
self.old_stdout.flush(); self.old_stderr.flush()
sys.stdout, sys.stderr = self._stdout, self._stderr
def __exit__(self, exc_type, exc_value, traceback):
self._stdout.flush(); self._stderr.flush()
sys.stdout = self.old_stdout
sys.stderr = self.old_stderr
if __name__ == '__main__':
devnull = open('log.txt', 'w')
#replaces sys.stdout, sys.stderr, see http://stackoverflow.com/a/6796752/2901002
with RedirectStdStreams(stdout=devnull, stderr=devnull):
df = pd.read_csv('data.csv', warn_bad_lines=True, error_bad_lines=False)
I can't help you with older than Python 3, but I've had very good success with the following:
import pandas as pd
from contextlib import redirect_stderr
import io
# Redirect stderr to something we can report on.
f = io.StringIO()
with redirect_stderr(f):
df = pd.read_csv(
new_file_name, header=None, error_bad_lines=False, warn_bad_lines=True, dtype=header_types
)
if f.getvalue():
logger.warning("Had parsing errors: {}".format(f.getvalue()))
I searched for this issue a number of times and kept being pointed to this questions. Hope it helps someone else, later on.
I am trying to use the call_command method to call the dumpdata command. Manually, I use it as follows to save the data to a file.
python manage.py dumpdata appname_one appname_two > /path/to/save/file.json
and it saves the json file. Now, I am in a situation where I need to call this command using the call_command method.
I am able to print out the json from the command using the following:
from django.core.management import call_command
call_command('dumpdata', 'appname_one', 'appname_two')
Is there a way I can save the given data to a file like we do it from the command line?
had to redirect sys.stdout to the file in order to achieve the above. Something like.
import sys
from django.core.management import call_command
sysout = sys.stdout
sys.stdout = open('filename.json', 'w')
call_command('dumpdata', 'appname_one', 'appname_two')
sys.stdout = sysout
An even better way is to use Django's built-in stdout redirection for their command modules. See docs here.
If you want to manipulate the stream before sending it to a file, you can also pass it a StringIO buffer:
import os
from cStringIO import StringIO
from django.core import management
def create_fixture(app_name, filename):
buf = StringIO()
management.call_command('dumpdata', app_name, stdout=buf)
buf.seek(0)
with open(filename, 'w') as f:
f.write(buf.read())
I am using Django fixture magic https://github.com/davedash/django-fixture-magic and need to dump a custom fixture. I tried several ways but ended up using Amyth's answer becuase it was the only way that worked.
Here is my admin action that works with fixture magic
def export_survey(modeladmin, request, queryset):
sysout = sys.stdout
survey = queryset[0]
fname = "%s.json" %(survey.slug)
response = HttpResponse(mimetype='application/json')
response['Content-Disposition'] = 'attachment; filename=%s' %(fname)
sys.stdout = response
call_command('custom_dump', 'complete_survey', survey.id)
sys.stdout = sysout
return response
export_survey.short_description = "Exports a single survey as a .json file"
DB fixtures typically compress well, and loaddata can read compressed fixtures. To write a .bz2 compressed fixture directly:
import bz2
with bz2.BZ2File('db.json.bz2', 'w', buffering=1024) as f:
django.core.management.call_command('dumpdata', stdout=f)
This one help for multiple dump data into json file
from django.core.management import call_command
import sys
sys.stdout = open('app_one/fixtures/apple.json', 'w')
call_command('dumpdata', 'app_one.apple')
sys.stdout = open('app_two/fixtures/banana.json', 'w')
call_command('dumpdata', 'app_two.banana')