I'm trying to make a recursive function by using os.listdir(), and I am having troble looping to all my directories and list out all the files and directories.
I know it's better using os.tree() for solving this kind of problem, but i want to see how to solve this by using os.listdir().
Here are my current code:
#!/bin/usr/py
from os.path import abspath
from os.path import isfile, isdir
import os
import sys
dir = sys.argv[1]
def recursive(dir):
files = os.listdir(dir)
for obj in files:
if isfile(obj):
print obj
elif isdir(obj):
print obj
recursive(abspath(obj))
#no idea why this won't work???
recursive(dir)
Your issue comes from abspath(obj), try replacing it by os.path.join(dir, obj) to have real path to your obj (I tested it on my env)
Thanks Gabriel and Emilrn ! this was exactly what I was looking for to recursively get the list of files from a parent directory provided for one of my projects. Just leaving the updated code here for someone who needs it later.
#!/bin/usr/py
import os
import sys
dir = sys.argv[1]
def recursive(dir):
files = os.listdir(dir)
for obj in files:
if os.path.isfile(os.path.join(dir,obj)):
print ("File : "+os.path.join(dir,obj))
elif os.path.isdir(os.path.join(dir,obj)):
recursive(os.path.join(dir, obj))
else:
print ('Not a directory or file %s' % (os.path.join(dir, obj))
recursive(dir)
Related
I am trying find if there is any files in the remote server match the provided pattern. Something as similar to the below solution
Airflow File Sensor for sensing files on my local drive
I used SSHOperator with bash command as below,
SSH_Bash = """
echo 'poking for files...'
ls /home/files/test.txt
if [ $? -eq "0" ]; then
echo 'Found file'
else
echo 'failed to find'
fi
"""
t1 = SSHOperator(
ssh_conn_id='ssh_default',
task_id='test_ssh_operator',
command=SSH_Bash,
dag=dag)
It works but doesnt look like an optimal solution. Could someone help me to get better solution than Bash script to sense the files in the remote server.
I tried the below sftp sensor ,
import os
import re
import logging
from paramiko import SFTP_NO_SUCH_FILE
from airflow.contrib.hooks.sftp_hook import SFTPHook
from airflow.operators.sensors import BaseSensorOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
class SFTPSensor(BaseSensorOperator):
#apply_defaults
def __init__(self, filepath,filepattern, sftp_conn_id='sftp_default', *args, **kwargs):
super(SFTPSensor, self).__init__(*args, **kwargs)
self.filepath = filepath
self.filepattern = filepattern
self.hook = SFTPHook(sftp_conn_id)
def poke(self, context):
full_path = self.filepath
file_pattern = re.compile(self.filepattern)
try:
directory = os.listdir(self.hook.full_path)
for files in directory:
if not re.match(file_pattern, files):
self.log.info(files)
self.log.info(file_pattern)
else:
context["task_instance"].xcom_push("file_name", files)
return True
return False
except IOError as e:
if e.errno != SFTP_NO_SUCH_FILE:
raise e
return False
class SFTPSensorPlugin(AirflowPlugin):
name = "sftp_sensor"
sensors = [SFTPSensor]
But this always poke into local machine instead of remote machine. Could someone help me where i am making a mistake.
I replaced the line from
directory = os.listdir(self.hook.full_path)
to
directory = self.hook.list_directory(full_path)
I have a function that first examines whether a txt file exists and if it does not it creates one. If the txt file already exists it reads the info. I am trying to write unittests to examine whether the logic of the function is correct. I want to patch things like existence of files, creation of files and reading of files.
The function to be tested looks like this:
import json
import os.path
def read_create_file():
filename = 'directory/filename.txt'
info_from_file = []
if os.path.exists(filename):
with open(filename, 'r') as f:
content = f.readlines()
for i in range(len(content)):
info_from_file.append(json.loads(content[i]))
return info_from_file
else:
with open(filename, 'w') as f:
pass
return []
The unittest looks like this:
import unittest
import mock
from mock import patch
class TestReadCreateFile(unittest.TestCase):
def setUp(self):
pass
def function(self):
return read_create_file()
#patch("os.path.exists", return_value=False)
#mock.patch('directory/filename.txt.open', new=mock.mock_open())
def test_file_does_not_exist(self, mock_existence, mock_open_patch):
result = self.function()
self.assertEqual(result, (True, []))
ERROR: ImportError: Import by filename is not supported.
or like this:
import unittest
import mock
from mock import patch
#patch("os.path.exists", return_value=False)
def test_file_not_exist_yet(self, mock_existence):
m = mock.mock_open()
with patch('__main__.open', m, create=True):
handle = open('directory/filename.txt', 'w')
result = self.function()
self.assertEqual(result, (True, {}))
ERROR:
IOError: [Errno 2] No such file or directory: 'directory/filename.txt'
As a newbie I cannot seem to get my head around a solution, any help is greatly appreciated.
Thank you
You're mocking os.path.exists wrong. When you patch you patch from the file under test.
#patch("path_to_method_under_test.path.exists", return_value=False)
def test_file_not_exist_yet(self, mock_existence):
I am writing some tests which use the python unittest package (Python 2.7)
and I heavily rely on the xmlrunner.XMLTestRunner to dump the XML test output
Unfortunately, I fail to find some basic example which describes how one can pass some command line options to the test class to parametrise some of the tests.
Does someone have some hint on how I could achieve this (using xmlrunner)?
In addition, here is what I try to achieve:
I define my tests in a set of classes in the following myunittest.py file:
import unittest
class TestOne(unittest.TestCase):
def __init__(self, options=None):
unittest.TestCase.__init__(self)
self.__options = options
def A(self):
print self.__options.configXML # try to print the parameter
self.assertEqual(1, 1)
and call it from the main.py which looks like:
from optparse import OptionParser
import unittest
import xmlrunner
from uitest import *
def runit(opt):
suite = unittest.TestSuite()
suite.addTest(TestOne(options=opt))
testrunner = xmlrunner.XMLTestRunner(output='tests', descriptions=True)
unittest.main(testRunner=testrunner).run(suite)
if __name__ == "__main__":
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("-c", "--configXML", dest="configXML", help="xml file")
options = parser.parse_args()
runit(opt=options)
Many thanks for your valuable help.
after several hours trying to figure it out, I came to this solution, which makes my day. I post it here, in case someone comes to the same issue.
The main drawback is that it seems I need to have everything in the same python file. and I run it like:
python test.py --xmlConfig=configFile.xml --xmlRunner
with test.py:
import unittest
import sys
from optparse import OptionParser
import xmlrunner
class MyTests(unittest.TestCase):
def testFirstThing(self):
xmlConfig=options.xmlConfig
self.assertEqual(xmlConfig,"configFile.xml")
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("--xmlRunner", "--xmlRunner", help="perform a unittest and generate XML", dest="xmlRunner", default=False, action='store_true')
parser.add_option("--xmlConfig", "--xmlConfig", type="string", help="configuration file", dest="xmlConfig", default="config.xml")
options, arguments = parser.parse_args()
if options.xmlRunner:
del sys.argv[1:]
unittest.main(testRunner=xmlrunner.XMLTestRunner(output='./xml'))
I have a code to rename file names which having extension ".dtshd".
import fnmatch
import os
import csv
import glob
with open('New_Names.csv') as f:
file_pattern = '*.dtshd*'
file_names = {}
reader = csv.reader(f)
for row in reader:
file_names[row[0]] = row[1]
for file in glob.glob(file_pattern):
path, filename = os.path.split(file)
filename_noext, ext = os.path.splitext(filename)
new_filename = file_names.get(filename_noext, filename_noext)
os.rename(os.path.join(path, filename),
os.path.join(path, '{}{}'.format(new_filename, '.bin')))
This is working fine.But i need to rename a file name with ".cpt" extension also.How do i add this in my code.Can you please guide me this.
Take a look at pathlib (it is not included the standard library in Python 2, though) and shutil.
from pathlib import Path
import shutil
for file in Path(".").glob(file_pattern):
shutil.move(str(file), str(file.with_suffix(".cpt")))
I have a less file which looks like below
#static_url: "/static/"; //don't know how to use static tag or {{STATIC_URL}}
#import url("#{static_url}site_common/bower_components/bootstrap/less/bootstrap.less");
#import url("#{static_url}momsplanner/custom_bootstrap/custom-variables.less");
#import url("#{static_url}momsplanner/custom_bootstrap/custom-other.less");
#import url("#{static_url}site_common/bower_components/bootstrap/less/utilities.less");
It works fine, but when I try to compile it with lessc, it's doable but very messy (i'll have to do collectstatic first, and give the STATIC_ROOT as lessc's include-path option)
I guess using relative path in the above less file is easier than that, are there other alternatives?
I'd advise using relative #imports. I wrote a helper function to construct the include paths from the configured django INSTALLED_APPS since we're using AppDirectoriesFinder that you could adapt to your manual trans-compilation process (we use django-compressor):
from compressor.filters.base import CompilerFilter
from django.utils.functional import memoize
_static_locations = {}
def _get_static_locations():
from django.conf import settings
"""
Captures all the static dirs (both from filesystem and apps) for build an include path for the LESS compiler.
"""
dirs = ['.']
for dir in settings.STATICFILES_DIRS:
dirs.append(dir)
for app in settings.INSTALLED_APPS:
from django.utils.importlib import import_module
import os.path
mod = import_module(app)
mod_path = os.path.dirname(mod.__file__)
location = os.path.join(mod_path, "static")
if os.path.isdir(location):
dirs.append(location)
return dirs
get_static_locations = memoize(_get_static_locations, _static_locations, 0)
class LessCompilerFilter(CompilerFilter):
def __init__(self, content, command=None, *args, **kwargs):
command = 'lessc --no-color --include-path=%s {infile} {outfile}' % ':'.join(get_static_locations())
super(LessCompilerFilter, self).__init__(content, command, *args, **kwargs)