Bazel : handle intermediate files in ctx.action - build

I am trying to implement custom c_library() rule in Bazel which takes a bunch of .c files as input and produces .a static library. In my implementation, .c files generate .o objects and then archives into .a library. But it seems that .o files are not getting generated.
Here is the custom c_library() rule (rules.bzl file):
def _change_ext_to_x( file_list, x = 'o' ):
# inputs a file list (i.e. string list)
# and changes their extensions to '.{x}'
return [ "".join( onefile.split('.')[:-1] ) + '.' + x \
for onefile in file_list ]
def _str_to_File( ctx, file_list ):
# A constructor of File() object
# pass the context
return [ ctx.new_file( onefile ) for onefile in file_list ]
def _c_library_impl( ctx ):
# implementation of 'c_library' rule
# the source files list ( strings )
in_file_list = []
for onefile in ctx.files.srcs:
in_file_list.append( onefile.basename )
out_file_list = _str_to_File( ctx, # the current context
_change_ext_to_x(in_file_list, x = 'o') )
ctx.action(
inputs = ctx.files.srcs,
outputs = out_file_list,
progress_message = "COMPILING FROM <.c> TO <.o>",
use_default_shell_env = True,
command = "gcc -c -O3 %s" % " ".join( in_file_list )
)
ctx.action(
inputs = out_file_list,
outputs = [ ctx.outputs._libfile ],
progress_message = "ARCHIVING <.o>s to <.a>",
use_default_shell_env = True,
arguments = [ctx.outputs._libfile.basename],
command = "ar cr $1 %s" % " ".join( [onefile.basename
for onefile in out_file_list] )
)
pass
c_library = rule(
# define rule for 'c' library
implementation = _c_library_impl,
attrs = {
"srcs" : attr.label_list( allow_files = True,
mandatory = True,
allow_empty = False ),
"out" : attr.output( mandatory = False )
},
outputs = {
"_libfile" : "lib%{name}.a"
}
)
And here is my BUILD file:
load("//:rules.bzl", "c_library")
c_library(
name = "foo",
srcs = glob(include = ["*.c"], exclude = ["main.c"])
# there are two files 'cmd.c' and 'utils.c' in the package
)
When I did bazel build //:foo, I got the following error:
INFO: Found 1 target...
ERROR: /home/spyder/Desktop/project/BUILD:3:1: output 'cmd.o' was not created.
ERROR: /home/spyder/Desktop/project/BUILD:3:1: output 'utils.o' was not created.
ERROR: /home/spyder/Desktop/project/BUILD:3:1: not all outputs were created or valid.
Target //:foo failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 2.249s, Critical Path: 1.94s
How exactly to persist the intermediate files between two successive ctx.actions ?

You should create one action per .c file instead of one action for all of them, that will also increase parallelism, and then specified the output:
def _c_library_impl( ctx ):
# implementation of 'c_library' rule
out_file_list = []
for f in ctx.files.srcs:
o = ctx.new_file(f.basename + ".o")
out_file_list.append(o)
ctx.action(
inputs = [f],
outputs = [o],
progress_message = "COMPILING FROM <.c> TO <.o>",
use_default_shell_env = True,
command = "gcc -c -O3 %s %s" % (f.path, o.path)
)
ctx.action(
inputs = out_file_list,
outputs = [ ctx.outputs._libfile ],
progress_message = "ARCHIVING <.o>s to <.a>",
use_default_shell_env = True,
command = "ar cr %s %s" % (ctx.outputs._libfile.path,
"\n".join([onefile.basename
for onefile in out_file_list] )
)

Related

how to play background music in Asterisk AGI while some process is Going on in background to remove the silence during execution

I am using python 2 in asterisk 2 there is a section where code listen to callers audio and process the audio. During the process there is a silence of 15 sec before the audio is played. I want to add a music during the processing of audio. Is there a way to do this.
the extension.config is like this
[autoattendant2]
exten => 5555,1,same=>n,Answer()
same=> n, AGI(/root/code_base/Queue/code2.py)
same=>n,hangup()
Below is the python code
#!/usr/bin/env python2
import sys
import re
import time
import random
import subprocess
import requests
import json
from datetime import datetime
env = {}
tests = 0;
while 1:
line = sys.stdin.readline().strip()
if line == '':
break
key,data = line.split(':')
if key[:4] <> 'agi_':
#skip input that doesn't begin with agi_
sys.stderr.write("Did not work!\n");
sys.stderr.flush()
continue
key = key.strip()
data = data.strip()
if key <> '':
env[key] = data
sys.stderr.write("AGI Environment Dump:\n");
sys.stderr.flush()
for key in env.keys():
sys.stderr.write(" -- %s = %s\n" % (key, env[key]))
sys.stderr.flush()
def checkresult (params):
params = params.rstrip()
if re.search('^200',params):
result = re.search('result=(\d+)',params)
if (not result):
sys.stderr.write("FAIL ('%s')\n" % params)
sys.stderr.flush()
return -1
else:
result = result.group(1)
#debug("Result:%s Params:%s" % (result, params))
sys.stderr.write("PASS (%s)\n" % result)
sys.stderr.flush()
return result
else:
sys.stderr.write("FAIL (unexpected result '%s')\n" % params)
sys.stderr.flush()
return -2
def change_file(path, cid):
# one of the process example
filename = 'complain{0}'.format(cid)
#filename =
input_file = path + '/' + filename + '.gsm'
output_file = path + '/' + filename + '.wav'
#command = "sox {} -r 8000 -c 1 {}".format(input_file, output_file)
command = "sox {} -b 16 -r 44100 -c 1 {} trim 0 7 vol 2".format(input_file, output_file)
subprocess.call(command, shell=True)
pbcheck = requests.get("http://127.0.0.1:8000/STT_complaint/", params = {"address" : output_file, "lang" : language, "cid":cid, "phone":callerid, "start_time":now})
res = pbcheck.content
res2 = res.replace('"', "")
return res2
def run_cmd(cmd):
#This runs the general command
sys.stderr.write(cmd)
sys.stderr.flush()
sys.stdout.write(cmd)
sys.stdout.flush()
result = sys.stdin.readline().strip()
checkresult(result)
#language = "ben"
# asking problem recorded audio
cmd_streaming = "STREAM FILE /root/code_base/recorded_voices/{0}/plz_tell_problem \"\"\n".format(language, language)
run_cmd(cmd_streaming)
# listening to caller / recording caller voice
cmd_record = "RECORD FILE {0}/complain{1} gsm 1234 {2} s=3 \"\"\n".format(fixed_path, cid, 15)
run_cmd(cmd_record)
#processing audio
processed_output = change_file(path , cid) # while this is executing ad giving output i want to play wait_music and then stop to run
# while processing play this
cmd_streaming = "STREAM FILE /root/code_base/recorded_voices/wait_music \"\"\n".format(language, language)
run_cmd(cmd_streaming)
# once output received (processed_output) play next audio
cmd_streaming = "STREAM FILE /root/code_base/recorded_voices/next_instruction \"\"\n")
run_cmd(cmd_streaming)
For that asterisk AGI have the special command "SET MUSIC ON"
https://wiki.asterisk.org/wiki/display/AST/Asterisk+18+AGICommand_set+music
Set waiting tone for asterisk agi function processing

error in readNetFromTensorflow in c++

I'm new in deep learning. In first step I create and train a model in python with keras and freezed by this code:
def export_model(MODEL_NAME, input_node_name, output_node_name):
tf.train.write_graph(K.get_session().graph_def, 'out', \
MODEL_NAME + '_graph.pbtxt')
tf.train.Saver().save(K.get_session(), 'out/' + MODEL_NAME + '.chkp')
freeze_graph.freeze_graph('out/' + MODEL_NAME + '_graph.pbtxt', None, \
False, 'out/' + MODEL_NAME + '.chkp', output_node_name, \
"save/restore_all", "save/Const:0", \
'out/frozen_' + MODEL_NAME + '.pb', True, "")
input_graph_def = tf.GraphDef()
with tf.gfile.Open('out/frozen_' + MODEL_NAME + '.pb', "rb") as f:
input_graph_def.ParseFromString(f.read())
output_graph_def = optimize_for_inference_lib.optimize_for_inference(
input_graph_def, [input_node_name], [output_node_name],
tf.float32.as_datatype_enum)
with tf.gfile.FastGFile('out/opt_' + MODEL_NAME + '.pb', "wb") as f:
f.write(output_graph_def.SerializeToString())
it's output :
checkpoint
Model.chkp.data-00000-of-00001
Model.chkp.index
Model.chkp.meta
Model_graph.pbtxt
frozen_Model.pb
opt_Model.pb
when I want to read the net in opencv c++ by readNetFromTensorflow :
String weights = "frozen_Model.pb";
String pbtxt = "Model_graph.pbtxt";
dnn::Net cvNet = cv::dnn::readNetFromTensorflow(weights, pbtxt);
This will make error :
OpenCV(4.0.0-pre) Error: Unspecified error (FAILED: ReadProtoFromBinaryFile(param_file, param). Failed to parse GraphDef file: frozen_Model.pb) in cv::dnn::ReadTFNetParamsFromBinaryFileOrDie, file D:\LIBS\OpenCV-4.00\modules\dnn\src\tensorflow\tf_io.cpp, line 44
and
OpenCV(4.0.0-pre) Error: Assertion failed (const_layers.insert(std::make_pair(name, li)).second) in cv::dnn::experimental_dnn_v4::`anonymous-namespace'::addConstNodes, file D:\LIBS\OpenCV-4.00\modules\dnn\src\tensorflow\tf_importer.cpp, line 555
How to fix this error?
Amin, may I ask you to try to save a graph in a testing mode:
K.backend.set_learning_phase(0) # <--- This setting makes all the following layers work in test mode
model = Sequential(name = MODEL_NAME)
model.add(Conv2D(filters = 128, kernel_size = (5, 5), activation = 'relu',name = 'FirstLayerConv2D_No1',input_shape = (Width, Height, image_channel)))
...
model.add(Dropout(0.25))
model.add(Dense(100, activation = 'softmax', name = 'endNode'))
# Create a graph definition (with no weights)
sess = K.backend.get_session()
sess.as_default()
tf.train.write_graph(sess.graph.as_graph_def(), "", 'graph_def.pb', as_text=False)
Then freeze your checkpoint files with a newly created graph_def.pb by a freeze_graph.py script (do not forget to use --input_binary flag).
Part of the code :
Create model, train and export_model
train_batch = gen.flow_from_directory(path + 'Train', target_size = (Width, Height), shuffle = False, color_mode = color_mode,
batch_size = batch_size_train, class_mode = 'categorical')
.
.
X_train, Y_train = next(train_batch)
.
.
X_train = X_train.reshape(X_train.shape).astype('float32')
.
.
model = Sequential(name = MODEL_NAME)
model.add(Conv2D(filters = 128, kernel_size = (5, 5), activation = 'relu',name = 'FirstLayerConv2D_No1',input_shape = (Width, Height, image_channel)))
model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation = 'relu'))
model.add(MaxPool2D(pool_size = (2, 2)))
model.add(BatchNormalization())
.
.
.
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(200, activation = 'tanh'))
model.add(BatchNormalization())
model.add(Dropout(0.25))
model.add(Dense(100, activation = 'softmax', name = 'endNode'))
model.compile(loss = 'categorical_crossentropy',
optimizer = SGD(lr = 0.01, momentum = 0.9), metrics = ['accuracy'])
history = model.fit(X_train, Y_train, batch_size = batch_size_fit, epochs = epoch, shuffle = True,
verbose = 1, validation_split = .1, validation_data = (X_test, Y_test))
export_model(MODEL_NAME, "FirstLayerConv2D_No1/Relu", "endNode/Softmax")
When you're writing the graph in python you need to do the following steps:
with tf.Session(graph=tf.Graph()) as sess:
# 1. Load saved model
saved_model = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], SAVED_MODEL_PATH)
# 2. Convert variables to constants
inference_graph_def = tf.graph_util.convert_variables_to_constants(sess, saved_model.graph_def, OUTPUT_NODE_NAMES)
# 3. Optimize for inference
optimized_graph_def = optimize_for_inference_lib.optimize_for_inference(inference_graph_def,
INPUT_NODE_NAMES,
OUTPUT_NODE_NAMES,
tf.float32.as_datatype_enum)
# 4. Save .pb file
tf.train.write_graph(optimized_graph_def, MODEL_DIR, 'model_name.pb', as_text=False)
# 5. Transform graph
transforms = [
'strip_unused_nodes(type=float, shape=\"1,128,128,3\")',
'remove_nodes(op=PlaceholderWithDefault)',
'remove_device',
'sort_by_execution_order'
]
transformed_graph_def = TransformGraph(optimized_graph_def,
INPUT_NODE_NAMES,
OUTPUT_NODE_NAMES,
transforms)
# 6. Remove constant nodes and attributes
for i in reversed(range(len(transformed_graph_def.node))):
if transformed_graph_def.node[i].op == "Const":
del transformed_graph_def.node[i]
for attr in ['T', 'data_format', 'Tshape', 'N', 'Tidx', 'Tdim',
'use_cudnn_on_gpu', 'Index', 'Tperm', 'is_training', 'Tpaddings']:
if attr in transformed_graph_def.node[i].attr:
del transformed_graph_def.node[i].attr[attr]
# 7. Save .pbtxt file
tf.train.write_graph(transformed_graph_def, MODEL_DIR, 'model_name.pbtxt', as_text=True)
Plus, if you have special nodes as Flatten, you need to remove and rename some nodes manually.
More info here.

python - ZeroDivisionError

I created a script which copy data to specific location. What i tried to do is print a results via progress-bar. I tried to use package : -> https://pypi.python.org/pypi/progressbar2
Here is my code:
src = raw_input("Enter source disk location: ")
src = os.path.abspath(src)
dst = raw_input("Enter first destination to copy: ")
dst = os.path.abspath(dst)
dest = raw_input("Enter second destination to move : ")
dest = os.path.abspath(dest)
for dir, dirs, files in os.walk(src):
if any(f.endswith('.mdi') for f in files):
dirs[:] = [] # do not recurse into subdirectories
continue # ignore this directory
files = [os.path.join(dir, f) for f in files]
progress, progress_maxval = 0, len(files) pbar = ProgressBar(widgets=['Progress ', Percentage(), Bar(), ' ', ETA(), ],maxval=progress_maxval).start()
debug_status = ''
for list in files:
part1 = os.path.dirname(list)
part2 = os.path.dirname(os.path.dirname(part1))
part3 = os.path.split(part1)[1]
path_miss1 = os.path.join(dst, "missing_mdi")
# ---------first location-------------------#
path_miss = os.path.join(path_miss1, part3)
# ---------second location-------------------#
path_missing = os.path.join(dest, "missing_mdi")
try:
# ---------first location-------------------#
if not os.path.exists(path_miss):
os.makedirs(path_miss)
else:
pass
if os.path.exists(path_miss):
distutils.dir_util.copy_tree(part1, path_miss)
else:
debug_status += "missing_file\n"
pass
if (get_size(path_miss)) == 0:
os.rmdir(path_miss)
else:
pass
# ---------second location-------------------#
if not os.path.exists(path_missing):
os.makedirs(path_missing)
else:
pass
if os.path.exists(path_missing):
shutil.move(part1, path_missing)
else:
debug_status += "missing_file\n"
if (get_size(path_missing)) == 0:
os.rmdir(path_missing)
else:
pass
except Exception:
pass
finally:
progress += 1
pbar.update(progress)
pbar.finish()
print debug_status
When i tried to execute it i got error and My Traceback is below:
Traceback (most recent call last):
File "<string>", line 254, in run_nodebug
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\CopyClass.py", in <module>
pbar = ProgressBar(widgets=['Progress ', Percentage(), Bar(), ' ', ETA(),],maxval=progress_maxval).start()
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\__init__.py", in start
self.update(0)
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\__init__.py", line 283, in update
self.fd.write(self._format_line() + '\r')
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\__init__.py", line 243, in _format_line
widgets = ''.join(self._format_widgets())
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\__init__.py", line 223, in _format_widgets
widget = format_updatable(widget, self)
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\widgets.py", in format_updatable
if hasattr(updatable, 'update'): return updatable.update(pbar)
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\widgets.py", in update
return '%3d%%' % pbar.percentage()
File "C:\Users\kostrzew\Desktop\REPORTS\ClassCopy\progressbar\__init__.py", line 208, in percentage
return self.currval * 100.0 / self.maxval
ZeroDivisionError: float division by zero
I know that there is a problem with "maxval=progress_maxval" because it can't be devided by zero.
My qestion is ,how to change it? Should i create exception to ignore zero ? How to do it ?
I think inside the ProgressBar its trying divide to zero. It calculates like this:
max_value - 100%
progress_value - x and from this formula if we find x? will be this:
x = (100 * progress_value) / max_value
for this solution set 1 instead of 0 for max_value.

Python 2 custom importer - import module vs import package

In the (legacy) code I maintain a custom importer is used:
class UnicodeImporter(object):
def find_module(self,fullname,path=None):
if isinstance(fullname,unicode):
fullname = fullname.replace(u'.',u'\\')
exts = (u'.pyc',u'.pyo',u'.py')
else:
fullname = fullname.replace('.','\\')
exts = ('.pyc','.pyo','.py')
if os.path.exists(fullname) and os.path.isdir(fullname):
return self
for ext in exts:
if os.path.exists(fullname+ext):
return self
def load_module(self,fullname):
if fullname in sys.modules:
return sys.modules[fullname]
else: # set to avoid reimporting recursively
sys.modules[fullname] = imp.new_module(fullname)
if isinstance(fullname,unicode):
filename = fullname.replace(u'.',u'\\')
ext, initfile = u'.py', u'__init__'
else:
filename = fullname.replace('.','\\')
ext, initfile = '.py', '__init__'
try:
if os.path.exists(filename+ext): # some foo.py
with open(filename+ext,'U') as fp:
mod = imp.load_source(fullname,filename+ext,fp)
sys.modules[fullname] = mod
mod.__loader__ = self
else: # some directory
mod = sys.modules[fullname]
mod.__loader__ = self
mod.__file__ = os.path.join(os.getcwd(),filename)
mod.__path__ = [filename]
#init file
initfile = os.path.join(filename,initfile+ext)
if os.path.exists(initfile):
with open(initfile,'U') as fp:
code = fp.read()
exec compile(code, initfile, 'exec') in mod.__dict__
return mod
except Exception as e: # wrap in ImportError a la python2 - will keep
# the original traceback even if import errors nest
print 'fail', filename+ext
raise ImportError, u'caused by ' + repr(e), sys.exc_info()[2]
if not hasattr(sys,'frozen'):
sys.meta_path = [UnicodeImporter()]
The code is here
I have two questions:
why are people using a different way for importing packages vs modules ? Can't I use imp.load_source() in the package path somehow ? imp.load_module() ? Would placement of mod.__loader__ = self matter in those cases ? Basically I am looking for "the one right way"
should this be guarded with imp.acquirelock() ?

replacing specific lines in a text file using python

First of all I am pretty new at python, so bear with me. I am attempting to read from one file, retrieve specific values and overwrite old values in another file with a similar format. The format is 'text value=xxx' in both files. I have the first half of the program working, I can extract the values I want and have placed them into a dict named 'params{}'. The part I haven't figured out is how to just write the specific value into the target file without it showing up at the end of the file or just writing garbage or only half of the file. Here is my source code so far:
import os, os.path, re, fileinput, sys
#set the path to the resource files
#res_files_path = r'C:\Users\n518013\Documents\203-104 WA My MRT Files\CIA Data\pelzer_settings'
tst_res_files_path = r'C:\resource'
# Set path to target files.
#tar_files_path = r'C:\Users\n518013\Documents\203-104 WA My MRT Files\CIA Data\CIA3 Settings-G4'
tst_tar_files_path = r'C:\target'
#test dir.
test_files_path = r'C:\Users\n518013\Documents\MRT Equipment - BY 740-104 WA\CIA - AS\Setting Files\305_70R_22_5 setting files\CIA 1 Standard'
# function1 to find word index and point to value
def f_index(lst, item):
ind = lst.index(item)
val = lst[ind + 3]
print val
return val
# function 2 for values only 1 away from search term
def f_index_1(lst, item):
ind = lst.index(item)
val = lst[ind + 1]
return val
# Create file list.
file_lst = os.listdir(tst_res_files_path)
# Traverse the file list and read in dim settings files.
# Set up dict.
params = {}
#print params
for fname in file_lst:
file_loc = os.path.join(tst_res_files_path, fname)
with open(file_loc, 'r') as f:
if re.search('ms\.', fname):
print fname
break
line = f.read()
word = re.split('\W+', line)
print word
for w in word:
if w == 'Number':
print w
params['sectors'] = f_index(word, w)
elif w == 'Lid':
params['lid_dly'] = f_index(word, w)
elif w == 'Head':
params['rotation_dly'] = f_index(word, w)
elif w == 'Horizontal':
tmp = f_index_1(word, w)
param = int(tmp) + 72
params['horizontal'] = str(param)
elif w == 'Vertical':
tmp = f_index_1(word, w)
param = int(tmp) - 65
params['vertical'] = str(param)
elif w == 'Tilt':
params['tilt'] = f_index_1(word, w)
else:
print 'next...'
print params #this is just for debugging
file_tar = os.path.join(tst_tar_files_path, fname)
for lines in fileinput.input(file_tar, inplace=True):
print lines.rstrip()
if lines.startswith('Number'):
if lines[-2:-1] != params['sectors']:
repl = params['sectors']
lines = lines.replace(lines[-2:-1], repl)
sys.stdout.write(lines)
else:
continue
Sample text files:
[ADMINISTRATIVE SETTINGS]
SettingsType=SingleScan
DimensionCode=
Operator=
Description=rev.1 4sept03
TireDimClass=Crown
TireWidth=400mm
[TEST PARAMETERS]
Number Of Sectors=9
Vacuum=50
[DELAY SETTINGS]
Lid Close Delay=3
Head Rotation Delay=3
[HEAD POSITION]
Horizontal=140
Vertical=460
Tilt=0
[CALIBRATION]
UseConvFactors=0
LengthUnit=0
ConvMMX=1
ConvPixelX=1
CorrFactorX=1
ConvMMY=1
ConvPixelY=1
CorrFactorY=1
end sample txt.
The code I have only writes about half of the file back, and I don't understand why? I am trying to replace the line 'Number of Sectors=9' with 'Number of Sectors=8' if I could get this to work, the rest of the replacements can be done using if statements.
Please help! I've spent hours on google looking for answers and info and everything I find gets me close but no cigar!
Thank you all in advance!
your file has the '.ini' format. python supports reading and writing those with the ConfigParser module. you could do this:
# py3: from pathlib import Path
import os.path
import configparser
# py3: IN_PATH = Path(__file__).parent / '../data/sample.ini'
# py3: OUT_PATH = Path(__file__).parent / '../data/sample_out.ini'
HERE = os.path.dirname(__file__)
IN_PATH = os.path.join(HERE, '../data/sample.ini')
OUT_PATH = os.path.join(HERE, '../data/sample_out.ini')
config = configparser.ConfigParser()
# py3: config.read(str(IN_PATH))
config.read(IN_PATH)
print(config['CALIBRATION']['LengthUnit'])
config['CALIBRATION']['LengthUnit'] = '27'
# py3: with OUT_PATH.open('w') as fle:
with open(OUT_PATH, 'w') as fle:
config.write(fle)