I am trying to create two files with the same data. One file to use for updating live web data and the other as a log. One file needs to be appended to and updated frequently. I can create the log fine but am struggling on how to handle the data for the second file.
I have tried using a 'with open' statement for the log file. When I try reading this into a live web page, it shows me the data that has been logged previously, and updates the data only when the file is closed.
#!/usr/bin/env python2.7
import os
import RPi.GPIO as GPIO
import time
import subprocess
#Solar Panel Script 1.0
#Set pin for Pump Relay Signal (PR = pin 29)
#Set up Pump Relay BCM5 (pin 29) as output pin in off position
GPIO.setmode(GPIO.BCM)
GPIO.setup (5, GPIO.OUT, initial=0)
GPIO.setwarnings(False)
#Load Hot Water Tank (HWT), Solar Panel (SP), and Outside Temp (OT) with OWFS
#Create CSV File for temperature data
from time import sleep, strftime, time
with open("/var/www/html/data.csv", "a") as log:
while True:
with open ("/mnt/1wire/28.C14777910F02/temperature", "r") as myfile:
HWT=myfile.read().replace('\n', '')
myfile.close()
with open ("/mnt/1wire/28.390877910402/temperature", "r") as myfile2:
SP=myfile2.read().replace('\n', '')
myfile.close()
log.write("{0},{1},{2}\n".format(strftime("%Y-%m-%d %H:%M:%S"), str(HWT), str(SP)))
#Solar Hot Water Heater Module
#Turns on PR only if SP is 10F hotter than HWT. Checks OT for frezing temps, if less than 33, PR is off.
print ('hot water: ' + HWT)
print ('solar panel: '+ SP)
flt_HWT = float(HWT)
flt_SP = float(SP)
if flt_HWT > 170:
GPIO.output(5, GPIO.LOW) #Pump Relay Off
if flt_SP > (flt_HWT + 10):
GPIO.output(5, GPIO.HIGH) #Pump Relay On
state = GPIO.input(5)
print state
sleep(20) #10 Minutes = 600
I expected the log file to allow me to collect data from it while it was open.
log.write("{0},{1},{2}\n".format(strftime("%Y-%m-%d %H:%M:%S"), str(HWT), str(SP)))
This is where you are writing the log. You can simply include another with open() statement here
with open("secondfile.log") as secfile:
log.write("{0},{1},{2}\n".format(strftime("%Y-%m-%d %H:%M:%S"), str(HWT), str(SP))) ##original log file can be here
secfile.write("{0},{1},{2}\n".format(strftime("%Y-%m-%d %H:%M:%S"), str(HWT), str(SP))) ##and here you are wrighting the second file.
However if you are wrighting multiple files it would be better to stick them into a function of their own.
def write_file(text, filename):
try:
with open(filename) as file:
file.write(text)
return True
except:
return False ##include any other exception stuff here
now you can use
success = write_file("log text", "filename.log")
if success:
success = write_file("log2 text", "filename2.log")
if success:
print("Yey both files have been written to")
else:
print("Awww, there was an error writing to the file")
Related
I'm trying to shorten or simplify my code.
I want to download a log file from an internal server which is updated every 10 seconds, but I'm only running my script every 10 or 15 minutes.
The log file is semicolon seperated and has many rows in it I don't use. So my workflow is as following.
get current date in YYYYMMDD format
download the file
delay for waiting that the file is finished downloading
trim the file to the rows I need
only process last line of the file
delete the files
I'm new to python and if you could help me to shorten/simplify my code in less steps I would be thankful.
import urllib
import time
from datetime import date
today = str(date.today())
import csv
url = "http://localserver" + today + ".log"
urllib.urlretrieve (url, "output.log")
time.sleep(15)
with open("output.log","rb") as source:
rdr= csv.reader(source, delimiter=';')
with open("result.log","wb") as result:
wtr= csv.writer( result )
for r in rdr:
wtr.writerow( (r[0], r[1], r[2], r[3], r[4], r[5], r[15], r[38], r[39], r[42], r[54], r[90], r[91], r[92], r[111], r[116], r[121], r[122], r[123], r[124]) )
with open('result.log') as myfile:
print (list(myfile)[-1]) #how do I access certain rows here?
You could probably make use of the advanced module, requests as below. The timeout can be increased depending on the time it takes for the download to complete successfully. Furthermore, the two with open statements can be consolidated in a single line. What is more, in order to load the line one by one in to the memory, we can make use of iter_lines generator. Note that stream=True should be set in order to load line one at a time.
from datetime import date
import csv
import requests
# Declare variables
today = str(date.today())
url = "http://localserver" + today + ".log"
outfile = 'output.log'
# Instead of waiting for 15 seconds explicitly consider using requests module
# with timeout parameter
response = requests.get(url, timeout=15, stream=True)
if response.status_code != 200:
print('Failed to get data:', response.status_code)
with open(outfile, 'w') as dest:
writer = csv.writer(dest)
# Walk through the request response line by line w/o loadin gto memory
line = list(response.iter_lines())[-1]
# Decode the response to string and split line by line
reader = csv.reader(line.decode('utf-8').splitlines(), delimiter=';')
# Read line by line for the splitted content and write to file
for r in reader:
writer.writerow((r[0], r[1], r[2], r[3], r[4], r[5], r[15], r[38], r[39], r[42], r[54], r[90], r[91], r[92],
r[111], r[116], r[121], r[122], r[123], r[124]))
print('File written successfully: ' + outfile)
I posted a question about how to catch a "sudo shutdown -r 2" event in Python. I was sent to this thread: Run code in python script on shutdown signal .
I'm running a Raspberry Pi v2 with Jessy.
I have read about
signal
and have tried to follow the ideas in the above thread, but so far I have not been successful. Here is my code:
import time
import signal
import sys
def CloseAll(Code, Frame):
f = open('/mnt/usbdrive/output/TestSignal.txt','a')
f.write('Signal Code:' + Code)
f.write('Signal Frame:' + Frame)
f.write('\r\n')
f.close()
sys.exit(0)
signal.signal(signal.SIGTERM,CloseAll)
print('Program is running')
try:
while True:
#get readings from sensors every 15 seconds
time.sleep(15)
f = open('/mnt/usbdrive/output/TestSignal.txt','a')
f.write('Hello ')
f.write('\r\n')
f.close()
except KeyboardInterrupt:
f = open('/mnt/usbdrive/output/TestSignal.txt','a')
f.write('Done')
f.write('\r\n')
f.close()
The program runs in a "screen" session/window and reacts as expected to a CNTL-C. However, when I exit the screen session, leaving the program running, and enter "sudo shutdown -r 2", the Pi reboots as expected after 2 minutes, but the TestSignal.txt file does not show that the signal.SIGTERM event was processed.
What am I doing wrong? Or better yet, how can I trap the shutdown event, usually initiated by a cron job, and close my Python program running in a screen session gracefully?
When you do not try to await such an event, but in a parallel session send SIGTERMto that process (e.g. by calling kill -15 $PID on the process id $PID of the python script running) , you should see an instructive error message ;-)
Also the comment about the mount point should be of interest after you repaired the python errors (TypeError: cannot concatenate 'str' and 'int' objects).
Try something like:
import time
import signal
import sys
LOG_PATH = '/mnt/usbdrive/output/TestSignal.txt'
def CloseAll(Code, Frame):
f = open(LOG_PATH, 'a')
f.write('Signal Code:' + str(Code) + ' ')
f.write('Signal Frame:' + str(Frame))
f.write('\r\n')
f.close()
sys.exit(0)
signal.signal(signal.SIGTERM, CloseAll)
print('Program is running')
try:
while True:
# get readings from sensors every 15 seconds
time.sleep(15)
f = open(LOG_PATH, 'a')
f.write('Hello ')
f.write('\r\n')
f.close()
except KeyboardInterrupt:
f = open(LOG_PATH, 'a')
f.write('Done')
f.write('\r\n')
f.close()
as a starting point. If this works somehow on your system why not rewrite some portions like:
# ... 8< - - -
def close_all(signum, frame):
with open(LOG_PATH, 'a') as f:
f.write('Signal Code:%d Signal Frame:%s\r\n' % (signum, frame))
sys.exit(0)
signal.signal(signal.SIGTERM, close_all)
# 8< - - - ...
Edit: To further isolate the error and adapt more to production like mode, one might rewrite the code like this (given that syslog is running on the machine, which it should, but I never worked on devices of that kind):
#! /usr/bin/env python
import datetime as dt
import time
import signal
import sys
import syslog
LOG_PATH = 'foobarbaz.log' # '/mnt/usbdrive/output/TestSignal.txt'
def close_all(signum, frame):
"""Log to system log. Do not spend too much time after receipt of TERM."""
syslog.syslog(syslog.LOG_CRIT, 'Signal Number:%d {%s}' % (signum, frame))
sys.exit(0)
# register handler for SIGTERM(15) signal
signal.signal(signal.SIGTERM, close_all)
def get_sensor_readings_every(seconds):
"""Mock for sensor readings every seconds seconds."""
time.sleep(seconds)
return dt.datetime.now()
def main():
"""Main loop - maybe check usage patterns for file resources."""
syslog.syslog(syslog.LOG_USER, 'Program %s is running' % (__file__,))
try:
with open(LOG_PATH, 'a') as f:
while True:
f.write('Hello at %s\r\n' % (
get_sensor_readings_every(15),))
except KeyboardInterrupt:
with open(LOG_PATH, 'a') as f:
f.write('Done at %s\r\n' % (dt.datetime.now(),))
if __name__ == '__main__':
sys.exit(main())
Points to note:
the log file for the actual measurements is separate from the logging channel for operational alerts
the log file handle is safeguarded in context managing blocks and in usual operation is just kept open
for alerting the syslog channel is used.
as a sample for the message routing the syslog.LOG_USER on my system (OS X) gives me in all terminals a message, whilst the syslog.LOG_ERR priority message in signal handler only targets the system log.
should be more to the point during shutdown hassle (not opening a file, etc.)
The last point (5.) is important in case all processes receive a SIGTERM during shutdown, i.e. all want to do something (slowing things down), maybe screenalso does not accept any buffered input anymore (or does not flush), note stdout is block buffered not line buffered.
The decoupling of the output channels, should also ease the eventual disappearance of the mount point of the measurement log file.
I have written following code I am able to print out the parsed values of Lat and lon but i am unable to write them to a file. I tried flush and also i tried closing the file but of no use. Can somebody point out whats wrong here.
import os
import serial
def get_present_gps():
ser=serial.Serial('/dev/ttyUSB0',4800)
ser.open()
# open a file to write gps data
f = open('/home/iiith/Desktop/gps1.txt', 'w')
data=ser.read(1024) # read 1024 bytes
f.write(data) #write data into file
f = open('/home/iiith/Desktop/gps1.txt', 'r')# fetch the required file
f1 = open('/home/iiith/Desktop/gps2.txt', 'a+')
for line in f.read().split('\n'):
if line.startswith('$GPGGA'):
try:
lat, _, lon= line.split(',')[2:5]
lat=float(lat)
lon=float(lon)
print lat/100
print lon/100
a=[lat,lon]
f1.write(lat+",")
f1.flush()
f1.write(lon+"\n")
f1.flush()
f1.close()
except:
pass
while True:
get_present_gps()
You're covering the error up by using the except: pass. Don't do that... ever. At least log the exception.
One error which it definitely covers is lat+",", which is going to fail because it's float+str and it's not implemented. But there may be more.
I am processing certain output binary files using sloginfo on QNX, I have used ftp/telnet/vmware to get to a point where I upload the binary from my machine to the vmware instance and then run the sloginfo command.
The issue is that binary files which need to be processed are of inconsistent size (ranging from 50mb to 200mb), and the time needed to process each of these files is different, thus making it impossible to determine the wait/sleep time required.
I need to know if sloginfo returns a value which can be used as a flag. I tried using tn.read_until(), without getting desired results.
#
import os, sys, telnetlib, time
from ftplib import FTP
def upload(ftp, filed):
ext = os.path.splitext(filed)[1]
if ext in (".txt", ".htm", ".html"):
ftp.storlines("STOR " + filed, open(filed))
else:
ftp.storbinary("STOR " + filed, open(filed, "rb"), 1024)
def gettext(ftp, filename, outfile=None):
# fetch a text file
if outfile is None:
outfile = sys.stdout
# use a lambda to add newlines to the lines read from the server
ftp.retrlines("RETR " + filename, lambda s, w=outfile.write: w(s+"\n"))
if __name__ == '__main__':
dbfile = "LOG1"
nonpassive = False
remotesite = '192.168.0.128'
ftp_port = '21'
tel_port = '23'
password = 'root'
ftp = FTP()
ftp.connect(remotesite, ftp_port)
ftp.login('root','root')
print 'Uploading the Log file... Please wait...'
upload (ftp, dbfile)
print 'File Uploaded Successfully...'
tn = telnetlib.Telnet(remotesite, tel_port)
tn.read_until("login: ")
tn.write('root' + "\n")
if password:
tn.write(password + "\n")
tn.write("sloginfo LOG1 >> LOG1.txt\n")
**#need to get more control on this sleep time**
time.sleep(300)
print 'Downloading text file...'
gettext(ftp, "LOG1.txt", open(r'LOG1.txt','wb'))
ftp.close()
tn.close()
tn.write("sloginfo LOG1 >> LOG1.txt\n") modified the above comment with tn.write ('sloginfo '+ strdbfile + '>> ' + strdbfiletxt+ '; echo Done!\n') and this has resolved the issue
Using LibVLC, I'm trying to save a stream while playing it. This is the python code:
import os
import sys
import vlc
if __name__ == '__main__':
filepath = <either-some-url-or-local-path>
movie = os.path.expanduser(filepath)
if 'http://' not in filepath:
if not os.access(movie, os.R_OK):
print ( 'Error: %s file is not readable' % movie )
sys.exit(1)
instance = vlc.Instance("--sub-source marq --sout=file/ps:example.mpg")
try:
media = instance.media_new(movie)
except NameError:
print ('NameError: % (%s vs Libvlc %s)' % (sys.exc_info()[1],
vlc.__version__, vlc.libvlc_get_version()))
sys.exit(1)
player = instance.media_player_new()
player.set_media(media)
player.play()
#dont exit!
while(1):
continue
It saves the video stream to a file example.mpg. As per this doc, the command to save a stream is this :
--sout=file/ps:example.mpg
which I've using when creating an instance of vlc.Instance:
instance = vlc.Instance("--sub-source marq --sout=file/ps:example.mpg")
But the problem is that it only saves the stream, it doesn't play the stream simultaneously.
Is there any way (in LibVLC) I can save the stream (to a local file) while paying it?
Although, I'm looking for a solution in Python 3.3.1 but it is fine if there is any C or C++ solution.
I've created a similar, but not duplicate, topic yesterday.
Idea:
The basic idea is simple enough. You have to duplicate the output stream and redirect it to a file. This is done, as Maresh correctly pointed out, using the sout=#duplicate{...} directive.
Working Solution:
The following solution works on my machine ™. I've tested it on Ubuntu 12.10 with VLC v2.0.3 (TwoFlower) and Python 2.7.1. I think it should also work on Python 3 since most of the heavy lifting is done by libVlc anyway.
import os
import sys
import vlc
if __name__ == '__main__':
#filepath = <either-some-url-or-local-path>
movie = os.path.expanduser(filepath)
if 'http://' not in filepath:
if not os.access(movie, os.R_OK):
print ( 'Error: %s file is not readable' % movie )
sys.exit(1)
instance = vlc.Instance("--sout=#duplicate{dst=file{dst=example.mpg},dst=display}")
try:
media = instance.media_new(movie)
except NameError:
print ('NameError: % (%s vs Libvlc %s)' % (sys.exc_info()[1],
vlc.__version__, vlc.libvlc_get_version()))
sys.exit(1)
player = instance.media_player_new()
player.set_media(media)
player.play()
#dont exit!
while(1):
continue
Helpful Links
The Command-Line help was essential to decipher the plethora of VLCs
command line options.
Chapter 3 of VLC streaming HowTo. Explains the structure of the stream output, its directives and describes of the various available modules. Chapter 4 shows some examples.
LibVLC API documentation in case you want to change media option at
runtime
Update - Saving YouTube videos:
The above code doesn't play nice with YouTube. I searched around and discovered that an additional transcode directive can be used to convert YouTube's video stream to a regular video format. I used #transcode{vcodec=mp4v,acodec=mpga,vb=800,ab=128,deinterlace}
vcodec=mp4v is the video format you want to encode in (mp4v is MPEG-4, mpgv is MPEG-1, and there is also h263, DIV1, DIV2, DIV3, I420, I422, I444, RV24, YUY2).
acodec=mpga is the audio format you want to encode in (mpga is MPEG audio layer 2, a52 is A52 i.e. AC3 sound).
vb=800 is the video bitrate in Kbit/s.
ab=128 is the audio bitrate in Kbit/s.
deinterlace tells VLC to deinterlace the video on the fly.
The updated code looks like this:
import os
import sys
import vlc
if __name__ == '__main__':
#filepath = <either-some-url-or-local-path>
filepath = "http://r1---sn-nfpnnjvh-1gil.c.youtube.com/videoplayback?source=youtube&newshard=yes&fexp=936100%2C906397%2C928201%2C929117%2C929123%2C929121%2C929915%2C929906%2C929907%2C929125%2C929127%2C925714%2C929917%2C929919%2C912512%2C912515%2C912521%2C906838%2C904485%2C906840%2C931913%2C904830%2C919373%2C933701%2C904122%2C932216%2C936303%2C909421%2C912711%2C907228%2C935000&sver=3&expire=1373237257&mt=1373214031&mv=m&ratebypass=yes&id=1907b7271247a714&ms=au&ipbits=48&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&itag=45&key=yt1&ip=2a02%3A120b%3Ac3c6%3A7190%3A6823%3Af2d%3A732c%3A3577&upn=z3zzcrvPC0U&cp=U0hWSFJOVV9KUUNONl9KSFlDOmt4Y3dEWFo3dDFu&signature=D6049FD7CD5FBD2CC6CD4D60411EE492AA0E9A77.5D0562CCF4E10A6CC53B62AAFFF6CB3BB0BA91C0"
movie = os.path.expanduser(filepath)
savedcopy = "yt-stream.mpg"
if 'http://' not in filepath:
if not os.access(movie, os.R_OK):
print ( 'Error: %s file is not readable' % movie )
sys.exit(1)
instance = vlc.Instance("--sout=#transcode{vcodec=mp4v,acodec=mpga,vb=800,ab=128,deinterlace}:duplicate{dst=file{dst=%s},dst=display}" % savedcopy)
try:
media = instance.media_new(movie)
except NameError:
print ('NameError: % (%s vs Libvlc %s)' % (sys.exc_info()[1],
vlc.__version__, vlc.libvlc_get_version()))
sys.exit(1)
player = instance.media_player_new()
player.set_media(media)
player.play()
#dont exit!
while(1):
continue
A couple of important points:
I've used MPEG audio and video codecs in the transcode directive. It seems to be important to use a matching extensions for the output file (mpg in this case). Otherwise VLC gets confused when opening the saved file for playback. Keep that in mind if you decide to switch to another video format.
You cannot add a regular YouTube URL as filepath. Instead you have to specify the location of the video itself. That's the reason why the filepath that I've used looks so cryptic. That filepath corresponds to video at http://www.youtube.com/watch?v=GQe3JxJHpxQ. VLC itself is able to extract the video location from a given YouTube URL, but libVLC doesn't do that out of the box. You'll have to write your own resolver to do that. See this related SO question. I followed this approach to manually resolve the video location for my tests.
I think you need to duplicate the output in order to play and record it at the same time:
vlc.Instance("--sub-source marq --sout=#stream_out_duplicate{dst=display,dst=std{access=file,mux=ts,dst=/path/file.mpg}}")
or
libvlc_media_add_option(media, ":sout=#stream_out_duplicate{dst=display,dst=std{access=file,mux=ts,dst=/path/file.mpg}}")
Did you try adding to the list of options the following option?
--sout-display
i.e.
instance = vlc.Instance("--sub-source marq --sout=file/ps:example.mpg --sout-display")
Some time ago in a sample code in the active state website i saw someone played and recorded a MP3 file using VLC using the vlc.py module. You can take a look at it's sample code to see how to duplicate a stream. I copied th code here for you (I copied it from http://code.activestate.com/recipes/577802-using-vlcpy-to-record-an-mp3-and-save-a-cue-file/):
import vlc
import time
import os
def new_filename(ext = '.mp3'):
"find a free filename in 00000000..99999999"
D = set(x[:8] for x in os.listdir('.')
if (x.endswith(ext) or x.endswith('.cue')) and len(x) == 12)
for i in xrange(10**8):
s = "%08i" %i
if s not in D:
return s
def initialize_cue_file(name,instream,audiofile):
"create a cue file and write some data, then return it"
cueout = '%s.cue' %name
outf = file(cueout,'w')
outf.write('PERFORMER "%s"\n' %instream)
outf.write('TITLE "%s"\n' %name)
outf.write('FILE "%s" WAVE\n' %audiofile)
outf.flush()
return outf
def initialize_player(instream, audiofile):
"initialize a vlc player which plays locally and saves to an mp3file"
inst = vlc.Instance()
p = inst.media_player_new()
cmd1 = "sout=#duplicate{dst=file{dst=%s},dst=display}" %audiofile
cmd2 ="no-sout-rtp-sap"
cmd3 = "no-sout-standard-sap"
cmd4 ="sout-keep"
med=inst.media_new(instream,cmd1,cmd2,cmd3,cmd4)
med.get_mrl()
p.set_media(med)
return p, med
def write_track_meta_to_cuefile(outf,instream,idx,meta,millisecs):
"write the next track info to the cue file"
outf.write(' TRACK %02i AUDIO\n' %idx)
outf.write(' TITLE "%s"\n' %meta)
outf.write(' PERFORMER "%s"\n' %instream)
m = millisecs // 60000
s = (millisecs - (m*60000)) // 1000
hs = (millisecs - (m*60000) - (s*1000)) //10
ts = '%02i:%02i:%02i' %(m,s,hs)
outf.write(' INDEX 01 %s\n' %ts)
outf.flush()
def test():
#some online audio stream for which this currently works ....
instream = 'http://streamer-mtc-aa05.somafm.com:80/stream/1018'
#if the output filename ends with mp3 vlc knows which mux to use
ext = '.mp3'
name = new_filename(ext)
audiofile = '%s%s' %(name,ext)
outf = initialize_cue_file(name,instream,audiofile)
p,med = initialize_player(instream, audiofile)
p.play()
np = None
i = 0
while 1:
time.sleep(.1)
new = med.get_meta(12)
if new != np:
i +=1
t = p.get_time()
print "millisecs: %i" %t
write_track_meta_to_cuefile(outf,instream,i,new,t)
np = new
print "now playing: %s" %np
if __name__=='__main__':
test()
Perhaps you need to clone your output, as suggested on the forum?