I want to unit test a piece of code from a module using the Mock testing library.
# File_name: do.py
assert __name__ == "__main__", "This module should NOT be imported."
from disk import Disk
def bool_fun(args):
res = Disk.disks()
print(res)
if args not in res:
return -1
return args
The above file contains assert name == "__main__" which is blocking the module from getting import in test file.
# File_name: do_test.py
import unittest
from unittest import mock
import do
class DOTest(unittest.TestCase):
#mock.patch('do.Disk.disks')
def test_bool_fun(self, mock_disks):
args = 4
mock_disks.return_value = [4,3,2,3,4]
res = do.bool_fun(args)
self.assertEqual(res, args)
if __name__ == "__main__":
unittest.main()
One more helper file which is imported in do.py file,
# File_name: disk.py
import random
class Disk():
def __init__(self):
pass
def disks():
lis = []
for i in range(5):
lis.append(random.randint(1,10))
return lis
Assuming all files are kept in the same directory, I'm getting below error when trying to test it.
assert __name__ == "__main__", "This module should NOT be imported."
AssertionError: This module should NOT be imported.
However, it works great if I remove the assert code from do.py file.
Can you please tell me how to test such files containing assert name == "__main__" in it.
Doing a lot of research and following over 100+ Stackoverflow links I can not find a solution that can straight away help in importing the file to UT.
Reason
You can mock the assert or the name because it hasn't been imported yet. You have to import it to assert, which won't work because you'll hit the exception.
Solution
So a hacky way, which should be OK since this is a UT, is to do an os.system call to "sed -i '/assert __name__/d' path/to/file" before importing it. That will just remove the line from the file at the run time :)
Example
import os
os.system("sed -i '/assert __name__/d' /path/to/file")
Related
I'm still learning Python so I apologize up front. I am trying to write a watcher that watches for the creation of a file that matches a specific pattern. I am using watchdog with some code that is out there to get it watch a directory.
I don't quite understand how to have the watcher watch for a filename that matches a pattern. I've edited the regexes=[] field with what I think might work but I have not had luck.
Here is an example of a file that might come in: Hires For Monthly TM_06Jan_0946.CSV
I can get the watcher to tell me when a .CSV is created in the directory, I just can't get it to only tell me when essentially Hires.*\.zip is created.
I've checked the this link but have not had luck
How to run the RegexMatchingEventHandler of Watchdog correctly?
Here is my code:
import time
import watchdog.events
from watchdog.observers import Observer
from watchdog.events import RegexMatchingEventHandler
import re
import watchdog.observers
import time
import fnmatch
import os
class Handler(watchdog.events.RegexMatchingEventHandler):
def __init__(self):
watchdog.events.RegexMatchingEventHandler.__init__(self, regexes=['.*'], ignore_regexes=[], ignore_directories=False, case_sensitive=False)
def on_created(self, event):
print("Watchdog received created event - % s." % event.src_path)
def on_modified(self, event):
print("Watchdog received modified event - % s." % event.src_path)
if __name__ == "__main__":
src_path = r"C:\Users\Downloads"
event_handler = Handler()
observer = watchdog.observers.Observer()
observer.schedule(event_handler, path=src_path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join() ```
The reason why '.*' regex works and while the Hires.*\.zip regex doesn't is watchdog tries to match the full path with the regex. In terms of regex: r'Hires.*\.zip' will not give you a full match with "C:\Users\Downloads\Hires blah.zip"
Instead, try:
def __init__(self):
watchdog.events.RegexMatchingEventHandler.__init__(
self, regexes=[r'^.*?\.cvs$', r'.*?Hires.*?\.zip'],
ignore_regexes=[], ignore_directories=False, case_sensitive=False)
This will match with all the cvs files and the zip files starts with "Hires". The '.*?' matches with "C:\Users\Downloads" and the rest ensures file types, file names etc.
I am trying to unit test a script that reads a requirements.txt file and goes through each line installing it with pip. However it my unit test it doesn't seem to enter the loop because the self.requirements is empty. I tried setting it to self.setupvirtualenv.requirements to ['foobar'] but this doesn't work.
My question is how can I mock the self.requirements list so that it enters the for loop and the subprocess.call gets called?
def install_requirements(self):
self.requirements = open(self.requirements_path, 'r').readlines()
for req in self.requirements:
req = req.strip()
pip_command = r'pip install {}'.format(req)
subprocess.call(pip_command, shell=True)
My unit test:
import setupvirualenv
import unittest
from mock import patch, Mock
def test_install_requirements(self, mock_open, mock_subprocess_call):
self.setupvirtualenv.requirements = ['foobar']
self.setupvirtualenv.install_requirements()
mock_open.assert_called_once_with('foobar', 'r')
mock_subprocess_call.assert_called_once()
The failure message
AssertionError: Expected 'call' to have been called once. Called 0 times.
You should mock the open call, so that readlines() returns the dummy data.
It looks as though you may be missing the patch decorators from your test method. I have included what they should look like below.
from mock import patch
#patch('setupvirtualenv.subprocess.call')
#patch('setupvirtualenv.open')
def test_install_requirements(self, mock_open, mock_subprocess_call):
# Mock the call to open() and the return value from readlines()
mock_open.return_value.readlines.return_value = ['foobar\n']
self.setupvirtualenv.install_requirements()
mock_open.assert_called_once_with('foobar', 'r')
mock_subprocess_call.assert_called_once()
I have to monitor an XML file being written by a tool running all the day. But the XML file is properly completed and closed only at the end of the day.
Same constraints as XML stream processing:
Parse an incomplete XML file on-the-fly and trigger actions
Keep track of the last position within the file to avoid processing it again from the beginning
On answer of Need to read XML files as a stream using BeautifulSoup in Python, slezica suggests xml.sax, xml.etree.ElementTree and cElementTree. But no success with my attempts to use xml.etree.ElementTree and cElementTree. There are also xml.dom, xml.parsers.expat and lxml but I do not see support for "on-the-fly parsing".
I need more obvious examples...
I am currently using Python 2.7 on Linux, but I will migrate to Python 3.x => please also provide tips on new Python 3.x features. I also use watchdog to detect XML file modifications => Optionally, reuse the watchdog mechanism. Optionally support also Windows.
Please provide easy to understand/maintain solutions. If it is too complex, I may just use tell()/seek() to move within the file, use stupid text search in the raw XML and finally extract the values using basic regex.
XML sample:
<dfxml xmloutputversion='1.0'>
<creator version='1.0'>
<program>TCPFLOW</program>
<version>1.4.6</version>
</creator>
<configuration>
<fileobject>
<filename>file1</filename>
<filesize>288</filesize>
<tcpflow packets='12' srcport='1111' dstport='2222' family='2' />
</fileobject>
<fileobject>
<filename>file2</filename>
<filesize>352</filesize>
<tcpflow packets='12' srcport='3333' dstport='4444' family='2' />
</fileobject>
<fileobject>
<filename>file3</filename>
<filesize>456</filesize>
...
...
First test using SAX failed:
import xml.sax
class StreamHandler(xml.sax.handler.ContentHandler):
def startElement(self, name, attrs):
print 'start: name=', name
def endElement(self, name):
print 'end: name=', name
if name == 'root':
raise StopIteration
if __name__ == '__main__':
parser = xml.sax.make_parser()
parser.setContentHandler(StreamHandler())
with open('f.xml') as f:
parser.parse(f)
Shell:
$ while read line; do echo $line; sleep 1; done <i.xml >f.xml &
...
$ ./test-using-sax.py
start: name= dfxml
start: name= creator
start: name= program
end: name= program
start: name= version
end: name= version
Traceback (most recent call last):
File "./test-using-sax.py", line 17, in <module>
parser.parse(f)
File "/usr/lib64/python2.7/xml/sax/expatreader.py", line 107, in parse
xmlreader.IncrementalParser.parse(self, source)
File "/usr/lib64/python2.7/xml/sax/xmlreader.py", line 125, in parse
self.close()
File "/usr/lib64/python2.7/xml/sax/expatreader.py", line 220, in close
self.feed("", isFinal = 1)
File "/usr/lib64/python2.7/xml/sax/expatreader.py", line 214, in feed
self._err_handler.fatalError(exc)
File "/usr/lib64/python2.7/xml/sax/handler.py", line 38, in fatalError
raise exception
xml.sax._exceptions.SAXParseException: report.xml:15:0: no element found
Since yesterday I found the Peter Gibson's answer about the undocumented xml.etree.ElementTree.XMLTreeBuilder._parser.EndElementHandler.
This example is similar to the other one but uses xml.etree.ElementTree (and watchdog).
It does not work when ElementTree is replaced by cElementTree :-/
import time
import watchdog.events
import watchdog.observers
import xml.etree.ElementTree
class XmlFileEventHandler(watchdog.events.PatternMatchingEventHandler):
def __init__(self):
watchdog.events.PatternMatchingEventHandler.__init__(self, patterns=['*.xml'])
self.xml_file = None
self.parser = xml.etree.ElementTree.XMLTreeBuilder()
def end_tag_event(tag):
node = self.parser._end(tag)
print 'tag=', tag, 'node=', node
self.parser._parser.EndElementHandler = end_tag_event
def on_modified(self, event):
if not self.xml_file:
self.xml_file = open(event.src_path)
buffer = self.xml_file.read()
if buffer:
self.parser.feed(buffer)
if __name__ == '__main__':
observer = watchdog.observers.Observer()
event_handler = XmlFileEventHandler()
observer.schedule(event_handler, path='.')
try:
observer.start()
while True:
time.sleep(10)
finally:
observer.stop()
observer.join()
While the script is running, do not forget to touch one XML file, or simulate the on-the-fly writing using this one line script:
while read line; do echo $line; sleep 1; done <in.xml >out.xml &
For information, the xml.etree.ElementTree.iterparse does not seem to support a file being written. My test code:
from __future__ import print_function, division
import xml.etree.ElementTree
if __name__ == '__main__':
context = xml.etree.ElementTree.iterparse('f.xml', events=('end',))
for action, elem in context:
print(action, elem.tag)
My output:
end program
end version
end creator
end filename
end filesize
end tcpflow
end fileobject
end filename
end filesize
end tcpflow
end fileobject
end filename
end filesize
Traceback (most recent call last):
File "./iter.py", line 9, in <module>
for action, elem in context:
File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1281, in next
self._root = self._parser.close()
File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1654, in close
self._raiseerror(v)
File "/usr/lib64/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror
raise err
xml.etree.ElementTree.ParseError: no element found: line 20, column 0
Three hours after posting my question, no answer received. But I have finally implemented the simple example I was looking for.
My inspiration is from saaj's answer and is based on xml.sax and watchdog.
from __future__ import print_function, division
import time
import watchdog.events
import watchdog.observers
import xml.sax
class XmlStreamHandler(xml.sax.handler.ContentHandler):
def startElement(self, tag, attributes):
print(tag, 'attributes=', attributes.items())
self.tag = tag
def characters(self, content):
print(self.tag, 'content=', content)
class XmlFileEventHandler(watchdog.events.PatternMatchingEventHandler):
def __init__(self):
watchdog.events.PatternMatchingEventHandler.__init__(self, patterns=['*.xml'])
self.file = None
self.parser = xml.sax.make_parser()
self.parser.setContentHandler(XmlStreamHandler())
def on_modified(self, event):
if not self.file:
self.file = open(event.src_path)
self.parser.feed(self.file.read())
if __name__ == '__main__':
observer = watchdog.observers.Observer()
event_handler = XmlFileEventHandler()
observer.schedule(event_handler, path='.')
try:
observer.start()
while True:
time.sleep(10)
finally:
observer.stop()
observer.join()
While the script is running, do not forget to touch one XML file, or simulate the on-the-fly writing using the following command:
while read line; do echo $line; sleep 1; done <in.xml >out.xml &
I've written one small python script to fix the checksum of L3-4 protocols using scapy. When I'm running the script it is not taking command line argument or may be some other reason it is not generating the fix checksum pcap. I've verified the rdpcap() from scapy command line it is working file using script it is not getting executed. My program is
import sys
import logging
logging.getLogger("scapy").setLevel(1)
try:
from scapy.all import *
except ImportError:
import scapy
if len(sys.argv) != 3:
print "Usage:./ChecksumFixer <input_pcap_file> <output_pcap_file>"
print "Example: ./ChecksumFixer input.pcap output.pcap"
sys.exit(1)
#------------------------Command Line Argument---------------------------------------
input_file = sys.argv[1]
output_file = sys.argv[2]
#------------------------Get The layer and Fix Checksum-------------------------------
def getLayer(p):
for paktype in (scapy.IP, scapy.TCP, scapy.UDP, scapy.ICMP):
try:
p.getlayer(paktype).chksum = None
except: AttributeError
pass
return p
#-----------------------FixPcap in input file and write to output fi`enter code here`le----------------
def fixpcap():
paks = scapy.rdpcap(input_file)
fc = map(getLayer, paks)
scapy.wrpcap(output_file, fc)
The reason your function is not executed is that you're not invoking it. Adding a call to fixpcap() at the end of your script shall fix this issue.
Furthermore, here are a few more corrections & suggestions:
The statement following except Exception: should be indented as well, as follows:
try:
from scapy.all import *
except ImportError:
import scapy
Use argparse to parse command-line arguments.
Wrap your main code in a if __name__ == '__main__': block.
If I run the following command:
>python manage.py test
Django looks at tests.py in my application, and runs any doctests or unit tests in that file. It also looks at the __ test __ dictionary for extra tests to run. So I can link doctests from other modules like so:
#tests.py
from myapp.module1 import _function1, _function2
__test__ = {
"_function1": _function1,
"_function2": _function2
}
If I want to include more doctests, is there an easier way than enumerating them all in this dictionary? Ideally, I just want to have Django find all doctests in all modules in the myapp application.
Is there some kind of reflection hack that would get me where I want to be?
I solved this for myself a while ago:
apps = settings.INSTALLED_APPS
for app in apps:
try:
a = app + '.test'
__import__(a)
m = sys.modules[a]
except ImportError: #no test jobs for this module, continue to next one
continue
#run your test using the imported module m
This allowed me to put per-module tests in their own test.py file, so they didn't get mixed up with the rest of my application code. It would be easy to modify this to just look for doc tests in each of your modules and run them if it found them.
Use django-nose since nose automatically find all tests recursivelly.
Here're key elements of solution:
tests.py:
def find_modules(package):
"""Return list of imported modules from given package"""
files = [re.sub('\.py$', '', f) for f in os.listdir(os.path.dirname(package.__file__))
if f.endswith(".py") and os.path.basename(f) not in ('__init__.py', 'test.py')]
return [imp.load_module(file, *imp.find_module(file, package.__path__)) for file in files]
def suite(package=None):
"""Assemble test suite for Django default test loader"""
if not package: package = myapp.tests # Default argument required for Django test runner
return unittest.TestSuite([doctest.DocTestSuite(m) for m in find_modules(package)])
To add recursion use os.walk() to traverse module tree and find python packages.
Thanks to Alex and Paul. This is what I came up with:
# tests.py
import sys, settings, re, os, doctest, unittest, imp
# import your base Django project
import myapp
# Django already runs these, don't include them again
ALREADY_RUN = ['tests.py', 'models.py']
def find_untested_modules(package):
""" Gets all modules not already included in Django's test suite """
files = [re.sub('\.py$', '', f)
for f in os.listdir(os.path.dirname(package.__file__))
if f.endswith(".py")
and os.path.basename(f) not in ALREADY_RUN]
return [imp.load_module(file, *imp.find_module(file, package.__path__))
for file in files]
def modules_callables(module):
return [m for m in dir(module) if callable(getattr(module, m))]
def has_doctest(docstring):
return ">>>" in docstring
__test__ = {}
for module in find_untested_modules(myapp.module1):
for method in modules_callables(module):
docstring = str(getattr(module, method).__doc__)
if has_doctest(docstring):
print "Found doctest(s) " + module.__name__ + "." + method
# import the method itself, so doctest can find it
_temp = __import__(module.__name__, globals(), locals(), [method])
locals()[method] = getattr(_temp, method)
# Django looks in __test__ for doctests to run
__test__[method] = getattr(module, method)
I'm not up to speed on Djano's testing, but as I understand it uses automatic unittest discovery, just like python -m unittest discover and Nose.
If so, just put the following file somewhere the discovery will find it (usually just a matter of naming it test_doctest.py or similar).
Change your_package to the package to test. All modules (including subpackages) will be doctested.
import doctest
import pkgutil
import your_package as root_package
def load_tests(loader, tests, ignore):
modules = pkgutil.walk_packages(root_package.__path__, root_package.__name__ + '.')
for _, module_name, _ in modules:
try:
suite = doctest.DocTestSuite(module_name)
except ValueError:
# Presumably a "no docstrings" error. That's OK.
pass
else:
tests.addTests(suite)
return tests