UTF-8 to EBCDIC using iconv in Python-script on USS - python-2.7

I am trying to convert utf-8 files from a directory listing on USS into ebcdic files BEFORE getting them into z/OS datasets.
Using a helper function which I found on stackoverflow (thanks for this!) I can issue shell-commands from within the python script:
def r(cmd_line):
return Popen(cmd_line.split(), stdout=PIPE).communicate()[0]
With this I can allocate and populate mainframe datasets from USS-files, using
r("tso alloc DSNAME(...) etc.") # to allocate a mainframe DS and
r("tso oget ...") # to populate the mainframe DS
However: some files need to be converted first, which in a shellscript I would simply code with
iconv -f UTF-8 -t IBM-1141 $utf8_file > $ebcdic_file
and I am totally at a loss of how to do this in python (2.7)?
Can't ask anybody in my shop since python was newly installed and I am currently the only one interested in it. Anyone an idea? Thanks a lot in advance!

Although not in the true spirit of python, you can do what you want by wrapping USS commands in a python script. Here is an example:
#!/bin/env python
from cStringIO import StringIO
import os
import sys
def r(cmd):
import subprocess
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
def allocate_dataset(dsName):
name = "'" + dsName + "'"
out = r(['/bin/tso', 'alloc', 'ds(' + name + ')', 'space(6000 2000)', 'track', 'lrecl(80)',
'dsntype(library)', 'blksize(3200)', 'recfm(f b)', 'dir(2)', 'new'])
for line in out.split():
print line
def not_allocated(dsName):
name = "'" + dsName + "'"
out = r(['/bin/tsocmd', 'listds ' + name])
for line in out.split():
if "NOT IN CATALOG" in out:
return True
return False
def ascii_to_ebcdic(from_codepage, to_codepage, fileName):
os.system('iconv -f' + from_codepage + ' -t' + to_codepage + ' <' + fileName + ' >ebcdic_' + fileName)
def copy_to_dataset(fileName, dsName, memberName):
dsn = "//'" + dsName + '(' + memberName + ")'"
os.system('cp -T ' + fileName + ' "' + dsn + '"')
def main():
dsName = "HLQ.MY.PYTHON"
if not_allocated(dsName):
print("Allocating '" + dsName + "' data set")
allocate_dataset(dsName)
ascii_to_ebcdic("UTF-8", "IBM-1047", "test.txt")
copy_to_dataset("ebcdic_test.txt", "HLQ.MY.PYTHON", "TXT")
member = "//'HLQ.MY.PYTHON(TXT)'"
os.system('cat -v "' + member + '"')
main()

Related

How to email variable value using raspberry pi and smtplib

I am working on this project where i have 3 string variables which I want to email their values to an email address. I was able to email a plain text message; however, I couldn't include these variables values in. Below is my codes:
import serial
import smtplib
import time
serialport = serial.Serial('/dev/ttyUSB0', 115200, timeout = 0.5)
user= 'user#gmail.com'
password= 'password'
receiver= 'receiver#gmail.com
subject= 'Solar tracker status'
header = 'To: ' + email + '\n' + 'From: ' + email + '\n' + 'Subject: ' +
subject
while True:
line = serialport.readline()
result = line.find(";")
if result > 0:
str = line.split(";")
volt=str[0]
power=str[1]
temp=str[2]
body = "\n" + + "Voltage: " + volt + "\n" + "Power: " + power + "\n" + "Temp: " + temp
print header + '\n' + body
s=smtplib.SMTP('smtp.gmail.com',587)
s.ehlo()
s.starttls()
s.ehlo()
s.login(email, password)
s.sendmail(email, email, header + '\n\n' + body)
s.quit
When running this script, i got this error message :
File "testserial.py", line 23, in <module>
body = "\n" + + "Voltage: " + volt
TypeError: bad operand type for unary +: 'str'
I have tried converting the variable into string using str(volt), then got this error message:
File "testserial.py", line 23, in <module>
str(volt)
TypeError: 'list' object is not callable
I can't understand this because they are originally in string format since i was able to write them into a text file using %s without having to convert it.
I think i just don't know how to pass a variable into the body of the email.
Please help!

Can Python add the " character to a string

I have to paste 3000 url's a day that are unformatted
Can i set up code to convert the raw paste data to a string?
(Example raw data) - 13 Michael Way Cottees NSW 2017
(Example changed data) - "13 Michael Way Cottees NSW 2017"
I have tried
RAW_URL = 13 Michael Way Cottees NSW 2017 + " "
RAW_URL = str(13 HOADLEY ST MAWSON ACT 2607)
RAW_DATA = ' " ' + (13 HOADLEY ST MAWSON ACT 2607) + ' " '
I keep getting "invalid syntax" error and not having much luck with google.
Once it's done it will be folded into the below code, to replace the single input on PASTED_CRM_DATA to a list just below
import requests
import csv
from lxml import html
import time
import sys
text2search = '''RECENTLY SOLD'''
PASTED_CRM_DATA = "13 HOADLEY ST MAWSON ACT 2607"
URL_LIST = 'https://www.realestate.com.au/property/' + str(PASTED_CRM_DATA.replace(' ', '-').lower()),
with open('REA.csv', 'wb') as csv_file:
writer = csv.writer(csv_file)
for index, url in enumerate(URL_LIST):
page = requests.get(url)
print '\r' 'Scraping URL ' + str(index+1) + ' of ' + str(len(URL_LIST))+ ' ' + url,
if text2search in page.text:
tree = html.fromstring(page.content)
(title,) = (x.text_content() for x in tree.xpath('//title'))
(price,) = (x.text_content() for x in tree.xpath('//div[#class="property-value__price"]'))
(sold,) = (x.text_content().strip() for x in tree.xpath('//p[#class="property-value__agent"]'))
writer.writerow([title, price, sold])
Any input is appreciated
First of all you should understand what strings are in python
In your examples that you have tried
RAW_URL = 13 Michael Way Cottees NSW 2017 + " "
RAW_URL = str(13 HOADLEY ST MAWSON ACT 2607)
RAW_DATA = ' " ' + (13 HOADLEY ST MAWSON ACT 2607) + ' " '
Here the characters you try to use a string are interpreted as actual code. To make your intentions clear to the interpreter use single quotes ' around them. (or double quotes)
RAW_URL = '13 Micheal Way Cottees NSW 2017'
RAW_DATA = '13 HOADLEY SY MAWSON ACT 2607'
To apply quotes use either string concatanation
RAW_URL = '"' + '13 Micheal Way Cottees NSW 2017' + '"'
Tough im not sure what you mean with raw paste data. Where is the data copied from? Is it by hand or done in the program?

Working with Strings/Text, TypeError: 'NoneType' object is not callable

I am trying to put together a web scraper and came across with this error, which I have no idea how to amend, I was looking at errors with the same name, but don't see the similarity with mine.. I am quite new on this btw.
The code is the following,
import bs4 as bs
from urllib import urlopen as uReq
sauce = uReq('http://servicios.lanacion.com.ar/archivo-f11/02/2017-c30')
soup = bs.BeautifulSoup(sauce,'html.parser')
acumulados = soup.findAll('li',{'class':'acumulados'})
filename = 'LaNacion.csv'
f = open(filename,'w')
headers = "Título, Encabezado\n"
f.write(headers)
for acum in acumulados:
title = acumulados[0].a
encabezado = acumulados[0].p
f.write(title.replace(',',' ') + ',' + encabezado.replace(',',' ') + '\n')
f.close()
Thanks!
both title and encabezado are of the type bs4.element.Tag. title.replace is None.
you probably meant to do this:
f.write(str(title).replace(',',' ') + ',' +
str(encabezado).replace(',',' ') + '\n')

Python - write filename / filepath into a textfile

I have some filenames defined in a Python script with tkinter who look like this:
def save_file(self):
self.filename = tkFileDialog.asksaveasfilename(title="Save...", filetypes=([("Excel Workbook","*.xlsx")]))
At the end the filename of these variables should be written out into a text file:
text_out = open('output.txt', 'w')
text_out.write("The first filename is " + self.filename + " + '\n')
text_out.write("The first pathname is " + self.filelocation + '\n')
text_out.close()
But it doesn't work. Has anyone any ideas? I have also tried it with:
text_out.write("The first filename is " + str(self.filename) + " + '\n')
but without the expedted result.
I tried it from the command line and got this error message:
SyntaxError: EOL while scanning string literal
This appears to mean that Python doesn't like having '\n' at the end of a bunch of strings stuck together with +. Try treating the items as a list and joining them, like this:
text_out = open('output.txt', 'w')
filewrite = ''.join([
'The first filename is ', self.filename, '\n',
'The first pathname is ', self.filelocation, '\n'])
text_out.write(filewrite)
text_out.close()
I got this idea from the Python Cookbook, and it works for me.

Find and later delete zombie informatica objects

Is there a easy way to identify and clean unused informatica artifacts?
Context: In one of the projects, there are lot of zombie sessions / mappings etc, the creators have long gone.
I want to do the following:
List/Delete all sessions that are not associated with a workflow.
List/Delete all mappings that are not used in any session/wf.
List/Delete all source/target that were not used in any mapping.
List/Delete all workflows that were not run in the past one year.
Someone mentioned about using : Designer > Tools > Queries . I can't express the above 1/2/3/4 with the option given, can anyone shed some light?
Note:
I’m not looking for click one by one and find dependencies.
I’m not looking for download the whole plant as xml and search dependencies
one by one
As this is not easily achievable with PowerCenter itself, I tried to come up with some simple tool to solve it. For the full description and download link please go to this page.
Below you'll find the code published as requested in comments: Find and later delete zombie informatica objects
Feel free to use, share and improve :) Any code review will be also much appreciated.
import subprocess
import os
from subprocess import *
import platform
import sys
import getpass
import configparser
#global variables declarations
currentDir=''
pmrepPath=''
domainFile=''
def connect_to_repo(Repository,Domain,User,Host,Port, UserSecurityDomain):
#password = raw_input("Enter password for Repository: " + Repository + ", User: " + User)
password = getpass.getpass()
print "\nConnecting..."
if Domain != '':
RepoCommand="pmrep connect -r "+Repository+" -d "+Domain+" -n "+User + " -x " + password #+" -X DOMAIN_PWD"
else:
RepoCommand="pmrep connect -r "+Repository+" -n "+User + " -x " + password + " -h " + Host + " -o " + Port
if UserSecurityDomain != '':
RepoCommand += " -s " + UserSecurityDomain
RepoCommand=RepoCommand.rstrip()
p=subprocess.Popen(RepoCommand,stderr=subprocess.PIPE,stdin=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)
out,err=p.communicate()
if p.returncode or err:
print "Connection Failed"
print err.strip()
print out.strip()
sys.stdout.flush()
sys.stdin.flush()
else:
print "Connection Successful"
sys.stdout.flush()
sys.stdin.flush()
return p.returncode
def execute_pmrep_command(command, output_file_name, start_line, end_line, line_prefix):
if len(line_prefix)>0: line_prefix+=' '
out=open(output_file_name,'a')
return_code=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)
output,error=return_code.communicate()
for line in output.split('\r\n')[start_line:end_line]:
out.writelines(line_prefix + line + '\n')
out.close()
return
def check_platform():
global domainFile
global currentDir
global pmrepPath
global platForm
platForm=platform.system()
print "Platform recognized : "+platForm
## if not os.getenv('INFA_HOME', 'C:\\Informatica\\9.5.1'):
## print "INFA_HOME env_variable not set in your "+platForm+" platform."
## print "Please set INFA_HOME and continue."
## raw_input()
## sys.exit(0)
if not os.getenv('INFA_DOMAINS_FILE'):
print "INFA_DOMAINS_FILE env_variable not set in your "+platForm+" platform."
print "Please set INFA_DOMAINS_FILE and continue."
raw_input()
sys.exit(0)
## elif not os.getenv('DOMAIN_PWD', 'vic'):
## print "DOMAIN_PWD env variable not set in your "+platForm+" platform."
## print "Please set DOMAIN_PWD and continue."
## raw_input()
## sys.exit(0)
## else:
## if platForm == 'Windows':
## pmrepPath=os.getenv('INFA_HOME').strip()+"\clients\PowerCenterClient\client\\bin"
## elif platForm == 'Linux':
## pmrepPath=os.getenv('INFA_HOME').strip()+"/server/bin"
## currentDir=os.getcwd()
## domainFile=os.getenv('INFA_DOMAINS_FILE','C:\\Informatica\\9.5.1\\domains.infa').strip()
config = configparser.RawConfigParser()
config.optionxform = lambda option: option
config.read('InfaRepo_ListUnusedObjects.cfg')
infaDir = config.get('Common', 'infaDir').strip()
Repository = config.get('Common', 'Repository').strip()
Domain = config.get('Common', 'Domain').strip()
Host = config.get('Common', 'Host').strip()
Port = config.get('Common', 'Port').strip()
Folder = config.get('Common', 'Folder').strip()
User = config.get('Common', 'User').strip()
UserSecurityDomain = config.get('Common', 'UserSecurityDomain').strip()
objectTypeList = config.get('Common', 'objectTypeList').split(',')
if Domain != '':
print 'Domain provided, will be used to connect.'
else:
print 'Domain not provided, Host and Port will be used to connect.'
for i in range(len(objectTypeList)):
objectTypeList[i]=objectTypeList[i].strip()
currentDir=os.getcwd()
outputDir=currentDir+'\\UnusedObjectsReport'
###objectTypeList = ['mapplet', 'mapping', 'session', 'source', 'target', 'worklet']
##
##objectTypeList = ['target']
pmrepPath=infaDir.strip()+"\clients\PowerCenterClient\client\\bin"
os.chdir(pmrepPath)
outFile = outputDir + "\ListOfUnusedObjects.txt"
if not os.path.exists(os.path.dirname(outFile)):
os.makedirs(os.path.dirname(outFile))
print 'Output file: ' + outFile
open(outFile,'w').writelines("Domain : "+Domain+"\nRepository : "+Repository+"\nUserName : "+User+"\n")
open(outFile,'a').writelines("***************************"+"\n")
open(outFile,'a').writelines("LIST OF UNUSED OBJECTS:\n")
outBatchFile = outputDir + "\DeleteUnusedObjects.bat"
tempDir = outputDir + "\\temp"
if not os.path.exists(outputDir):
os.makedirs(outputDir)
if not os.path.exists(tempDir):
os.makedirs(tempDir)
for tempFile in os.listdir(tempDir):
os.remove(os.path.join(tempDir, tempFile))
print 'Output batch file: ' + outBatchFile
if Domain != '':
RepoCommand="pmrep connect -r "+Repository+" -d "+Domain+" -n "+User
else:
RepoCommand="pmrep connect -r "+Repository+" -n "+User + " -h " + Host + " -o " + Port
if UserSecurityDomain != '':
RepoCommand += " -s " + UserSecurityDomain
open(outBatchFile,'w').writelines(pmrepPath+"\\"+RepoCommand+"\n")
objectTypeCounter=0
return_code=connect_to_repo(Repository,Domain,User,Host,Port,UserSecurityDomain)
objDepDict={}
error = False
errorList = []
#check if repository connection is successfull
if return_code==0:
for objectType in objectTypeList:
objectTypeCounter+=1
print "Step {0} of {1}: {2}".format(objectTypeCounter, len(objectTypeList), objectType)
objectFile = tempDir + "\\" + objectType + ".txt"
open(objectFile,'w').writelines("")
objectDepFile = tempDir + "\\" + objectType + "_dep.txt"
open(objectDepFile,'w').writelines("")
execute_pmrep_command("pmrep listobjects -f " + Folder + " -o " + objectType, objectFile, 8, -4, '')
objectList=open(objectFile).readlines()
objectCounter=0
if len(objectList) == 0:
print '\tNo {0}s found'.format(objectType)
elif objectList[0][:3] == ' [[':
error=True
for line in objectList:
errorList += [line.replace('\n','')]
break
for line in objectList:
objectCounter+=1
fields=line.split(' ')
if len(fields) == 2:
objectType=fields[0]
objectName=fields[1][:-1]
else:
objectType=fields[0]
objectName=fields[2][:-1]
#if the object is non-reusable, it obviously is in some workflow, so skipp it
if fields[1] == 'non-reusable':
print "\t{0} {1} of {2}: {3} is not a reusable {4} - skipping".format(objectType, objectCounter, len(objectList), objectName, objectType)
continue
command = "pmrep listobjectdependencies -f " + Folder + " -n " + objectName + " -o " + objectType + " -p parents"
#print "Getting object dependencies for " + objectType + " " + objectName
print "\t{0} {1} of {2}: {3}".format(objectType, objectCounter, len(objectList), objectName)
execute_pmrep_command(command, objectDepFile, 8, -6, objectName)
#find unused objects
for fileLine in open(objectDepFile,'r').readlines():
line = fileLine.split(' ')
if len(line) == 3:
Name = line[0]
ParentType = line[1]
ParentName = line[2]
else:
Name = line[0]
ParentType = line[1]
ParentName = line[3]
try:
objDepDict[objectType + ': ' + Name]+=[ParentType + ' ' + ParentName]
except:
objDepDict[objectType + ': ' + Name]=[ParentType + ' ' + ParentName]
found = False
for objectKey in objDepDict.iterkeys():
objectType, objName = objectKey.replace(' ','').split(':')
if len(objDepDict[objectKey]) <= 1:
if not found:
print '\n'
print 'Following unused objects have been found:'
found = True
print '\t{0}: {1}'.format(objectType, objName)
open(outFile,'a').writelines('{0}: {1}\n'.format(objectType, objName))
open(outBatchFile,'a').writelines("rem {0}\\pmrep deleteobject -f {1} -o {2} -n {3}\n".format(pmrepPath, Folder, objectType, objName))
execute_pmrep_command('pmrep cleanup', 'log.txt', 0, 0, '')
open(outBatchFile,'a').writelines(pmrepPath+'\\pmrep cleanup' + '\n')
if not error:
if not found:
print 'No unused objects found.'
print '\nDone.'
print 'Output file: ' + outFile
print 'Output batch file: ' + outBatchFile
else:
print 'Following errors occured:'
for e in errorList:
print '\t', e
#wait for key press
print '\nHit Enter to quit...'
raw_input()
#End of program