Error while using offline experiences for DDPG. custom environment dimensions (action space and state space) seem to be inconsistent with what is expected in DDPG RLLIB trainer.
Ubuntu, Ray 0.7 version (latest ray), DDPG example, offline dataset.
Used sampler builder for offline dataset.
Estimated DQN with this experience data and it ran through. Changed environment action space to be continuous (Box(,1)) and DDPG did not work.
from ray.tune.registry import register_env
TRAIN_BATCH_SIZE = 512
class mmt_ctns_offline_logs(gym.Env):
def __init__(self):
self.action_space = Box(0,50,shape=(,1), dtype=np.float32) #one dimension action space, values range 0 to 50 max
self.observation_space = Box(-100000, 100000, shape=(,58), dtype=np.float32) #58 columns in state space
register_env("mmt_env_ctnaction", lambda config: mmt_ctns_offline_logs()) #register custom environment
#define the configuration. Some of these are defaults. But I have explicitely defined them for clarify (within my team)
config_dict = {"env": "mmt_env_ctnaction", "evaluation_num_episodes":50, "num_workers": 11, "sample_batch_size": 512,
"train_batch_size": TRAIN_BATCH_SIZE,
"input": "<experience_replay_folder>/",
"output": "<any_folder>", "gamma": 0.99,
"horizon": None,
"optimizer_class": "SyncReplayOptimizer",
"optimizer": {"prioritized_replay":True},
"actor_hiddens": [128, 64], "actor_hidden_activation": "relu",
"critic_hiddens": [64, 64], "critic_hidden_activation": "relu", "n_step": 1,
"target_network_update_freq": 500,
"input_evaluation": [],
"ignore_worker_failures":True, 'log_level': "DEBUG",
"buffer_size": 50000,
"prioritized_replay": True,
"prioritized_replay_alpha": 0.6,
"prioritized_replay_beta": 0.4,
"prioritized_replay_eps": 1e-6,
"compress_observations": False,
"lr": 1e-3,
"actor_loss_coeff": 0.1,
"critic_loss_coeff": 1.0,
"use_huber": False,
"huber_threshold": 1.0,
"l2_reg": 1e-6,
"grad_norm_clipping": True,
"learning_starts": 1500,
}
config = ddpg.DEFAULT_CONFIG.copy() #dqn.DEFAULT_CONFIG.copy()
for k,v in config_dict.items():
config[k] = v
config_ddpg = config
config_ddpg
run_experiments({
'NM_testing_DDPG_offpolicy_noIS': {
'run': 'DDPG',
'env': 'mmt_env_ctnaction',
'config': config_ddpg,
'local_dir': "/oxygen/narasimham/ray/tmp/mmt/mmt_user_27_DDPG/"
},
})
Expected results from DDPG iterations.
Actual - ERROR:-
ray.exceptions.RayTaskError: ray_DDPGTrainer:train() (pid=89635, host=ip-10-114-53-179)
File "/home/ubuntu/anaconda3/envs/tf_p36n/lib/python3.6/site-packages/ray/rllib/utils/tf_run_builder.py", line 49, in get
self.feed_dict, os.environ.get("TF_TIMELINE_DIR"))
File "/home/ubuntu/anaconda3/envs/tf_p36n/lib/python3.6/site-packages/ray/rllib/utils/tf_run_builder.py", line 91, in run_timeline
fetches = sess.run(ops, feed_dict=feed_dict)
File "/home/ubuntu/anaconda3/envs/tf_p36n/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 877, in run
run_metadata_ptr)
File "/home/ubuntu/anaconda3/envs/tf_p36n/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1076, in _run
str(subfeed_t.get_shape())))
ValueError: Cannot feed value of shape (512,) for Tensor 'default_policy/action:0', which has shape '(?, 1)'
During handling of the above exception, another exception occurred:
Try with action space definition as follows:
self.action_space = Box(0,50,shape=(1,), dtype=np.float32)
Related
I think that the issue is silly.
I'd like to run the code on two computers and I need to use a list. I followed this Tutorials
I used my PC as a talker and computer of the robot as a listener.
when running the code on my PC, the output is good as I needed.
[INFO] [1574230834.705510]: [3.0, 2.1]
[INFO] [1574230834.805443]: [3.0, 2.1]
but once running the code on the computer of the robot, the output is:
Traceback (most recent call last):
File "/home/redhwan/learn.py", line 28, in <module>
talker()
File "/home/redhwan/learn.py", line 23, in talker
pub.publish(position.data)
File "/opt/ros/kinetic/lib/python2.7/dist-packages/rospy/topics.py", line 886, in publish
raise ROSSerializationException(str(e))
rospy.exceptions.ROSSerializationException: <class 'struct.error'>: 'required argument is not a float' when writing 'data: [3.0, 2.1]'
full code on PC:
#!/usr/bin/env python
import rospy
from std_msgs.msg import Float32
x = 3.0
y = 2.1
def talker():
# if a == None:
pub = rospy.Publisher('position', Float32, queue_size=10)
rospy.init_node('talker', anonymous=True)
# rospy.init_node('talker')
rate = rospy.Rate(10) # 10hz
while not rospy.is_shutdown():
position = Float32()
a = [x,y]
# a = x
position.data = list(a)
# position.data = a
# hello_str = [5.0 , 6.1]
rospy.loginfo(position.data)
pub.publish(position.data)
rate.sleep()
if __name__ == '__main__':
try:
talker()
except rospy.ROSInterruptException:
pass
full code on the computer of the robot:
#!/usr/bin/env python
import rospy
from std_msgs.msg import Float32
def callback(data):
# a = list(data)
a = data.data
print a
def listener():
rospy.init_node('listener', anonymous=True)
rospy.Subscriber("position", Float32, callback)
# spin() simply keeps python from exiting until this node is stopped
rospy.spin()
if __name__ == '__main__':
listener()
when using one number as float everything is OK.
I understand how to publish and subscribe to them separately as the float but I'd like to do it as list
Any ideas or suggestion, it would be appreciated.
When you exchange messages in ROS is preferred to adopt standard messages if there is something relatively simple. Of course, when you develop more sophisticated systems (or modules), you can implement your own custom messages.
So in the case of float array, Float32MultiArray is your friend.
Populating the message in one side will look like that (just an example using a 2 elements float32 array) in C++:
.
.
.
while (ros::ok())
{
std_msgs::Float32MultiArray velocities;
velocities.layout.dim.push_back(std_msgs::MultiArrayDimension());
velocities.layout.dim[0].label = "velocities";
velocities.layout.dim[0].size = 2;
velocities.layout.dim[0].stride = 1;
velocities.data.clear();
velocities.data.push_back(count % 255);
velocities.data.push_back(-(count % 255));
velocities_demo_pub.publish(velocities);
ros::spinOnce();
loop_rate.sleep();
++count;
}
.
.
.
in Python for 8 elements array an example will look like:
.
.
.
while not rospy.is_shutdown():
# compose the multiarray message
pwmVelocities = Float32MultiArray()
myLayout = MultiArrayLayout()
myMultiArrayDimension = MultiArrayDimension()
myMultiArrayDimension.label = "motion_cmd"
myMultiArrayDimension.size = 1
myMultiArrayDimension.stride = 8
myLayout.dim = [myMultiArrayDimension]
myLayout.data_offset = 0
pwmVelocities.layout = myLayout
pwmVelocities.data = [0, 10.0, 0, 10.0, 0, 10.0, 0, 10.0]
# publish the message and log in terminal
pub.publish(pwmVelocities)
rospy.loginfo("I'm publishing: [%f, %f, %f, %f, %f, %f, %f, %f]" % (pwmVelocities.data[0], pwmVelocities.data[1],
pwmVelocities.data[2], pwmVelocities.data[3], pwmVelocities.data[4], pwmVelocities.data[5],
pwmVelocities.data[6], pwmVelocities.data[7]))
# repeat
r.sleep()
.
.
.
and on the other side your callback (in C++), will look like:
.
.
.
void hardware_interface::velocity_callback(const std_msgs::Float32MultiArray::ConstPtr &msg) {
//velocities.clear();
if (velocities.size() == 0) {
velocities.push_back(msg->data[0]);
velocities.push_back(msg->data[1]);
} else {
velocities[0] = msg->data[0];
velocities[1] = msg->data[1];
}
vel1 = msg->data[0];
vel2 = msg->data[1];
//ROS_INFO("Vel_left: [%f] - Vel_right: [%f]", vel1 , vel2);
}
.
.
.
Hope that you got an idea...if you need something more drop me a line!
First, build a network with batch_norm
net = tf.layers.conv2d(inputs = features, filters = 64, kernel_size = [3, 3], strides = (2, 2), padding = 'same')
training = tf.Variable(False, name = 'training')
net = tf.contrib.layers.batch_norm(net, is_training = training)
net = tf.nn.relu(net)
net = tf.reshape(net, [-1, 64 * 7 * 7]) #
net = tf.layers.dense(inputs = net, units = class_num, kernel_initializer = tf.contrib.layers.xavier_initializer(), name = 'regression_output')
#......
#after training, save the graph and weights
sess.run(loss, feed_dict={features : train_imgs, x : real_delta, training : False})
saver = tf.train.Saver()
saver.save(sess, 'reshape_final.ckpt')
tf.train.write_graph(sess.graph.as_graph_def(), "", 'graph_final.pb')
After that, I freeze the graph->optimize>transform
python3 ~/.keras2/lib/python3.5/site-packages/tensorflow/python/tools/freeze_graph.py --input_graph=graph_final.pb --input_checkpoint=reshape_final.ckpt --output_graph=frozen_graph.pb --output_node_names=regression_output/BiasAdd
python3 ~/.keras2/lib/python3.5/site-packages/tensorflow/python/tools/optimize_for_inference.py --input frozen_graph.pb --output opt_graph.pb --frozen_graph True --input_names input --output_names regression_output/BiasAdd
~/Qt/3rdLibs/tensorflow/bazel-bin/tensorflow/tools/graph_transforms/transform_graph --in_graph=opt_graph.pb --out_graph=fused_graph.pb --inputs=input --outputs=regression_output/BiasAdd --transforms="fold_constants fold_batch_norms fold_old_batch_norms sort_by_execution_order"
Load the model
std::string const model("/home/ramsus/Qt/blogCodes2/deep_homography/cnn/tensorflow/fused_graph.pb");
dnn::Net net = dnn::readNetFromTensorflow(model);
if(net.empty()){
std::cerr<<"Can't load network by using the mode file:"<<std::endl;
std::cerr<<model<<std::endl;
throw std::runtime_error("net is empty");
}
it throw error messages:
BatchNorm/moments/mean:Mean(conv2d/convolution)(BatchNorm/moments/mean/reduction_indices)
keep_dims:[ ] Tidx:[ ] T:0 OpenCV Error: Unspecified error (Unknown
layer type Mean in op BatchNorm/moments/mean) in populateNet, file
/home/ramsus/Qt/3rdLibs/opencv/modules/dnn/src/tensorflow/tf_importer.cpp,
line 1077
/home/ramsus/Qt/3rdLibs/opencv/modules/dnn/src/tensorflow/tf_importer.cpp:1077:
error: (-2) Unknown layer type Mean in op BatchNorm/moments/mean in
function populateNet
How could I solve this issue?Thanks
1.Environment info
Operating System: Windows
Python version: Python 2.7.13
2.Error Message:
ValueError: No JSON object could be decoded
lgb_train = lgb.Dataset(X_train, y_train)
lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
params = {
'task':'train',
'boosting':'gbdt',
'objective':'binary',
'metric':{'l2', 'auc'},
'num_leaves': 62,
'learning_rate': 0.05,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'verbose': 20
}
gbm = lgb.train(params,
lgb_train,
num_boost_round=250,
valid_sets=lgb_eval)
print('Start predicting...')
y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
y_pred = np.round(y_pred)
print gbm.feature_importance()
Follow this link: https://github.com/Microsoft/LightGBM/issues/615. According to the contributor, this is a small bug: The infinite number cannot be handled by json.
I am trying to practice examples from PyNN website. The code is as follows
import pyNN.brian as p
rng = p.NumpyRNG(seed = 4242)
refractory_period = p.RandomDistribution('uniform', [2.0, 3.0], rng)
ctx_parameters = {'cm': 0.25, 'tau_m': 20.0, 'v_rest': -60, 'v_thresh': -50, 'tau_refrac': refractory_period, 'v_reset': -60, 'v_spike': -50.0, 'a': 1.0, 'b': 0.005, 'tau_w': 600, 'delta_T': 2.5,'tau_syn_E': 5.0, 'e_rev_E': 0.0, 'tau_syn_I': 10.0, 'e_rev_I': -80 }
tc_parameters = ctx_parameters.copy()
tc_parameters.update({'a': 20.0, 'b': 0.0})
thalamocortical_type = p.EIF_cond_exp_isfa_ista(**tc_parameters)
At this point i get an error saying:
Traceback (most recent call last):
File "/home/ruthvik/Desktop/Summer 2017/pynncheck.py", line 7, in <module>
thalamocortical_type = p.EIF_cond_exp_isfa_ista(**tc_parameters)
TypeError: __init__() got an unexpected keyword argument 'tau_refrac'
I actually checked the pyNN github page and i realized that there is in fact a class called EIF_cond_exp_isfa_ista and it also has the parameter 'tau_refrac'. I am not very comfortable with python classes and object orientation. It will be a great help if someone can guide through this.
Edit:
I defined c = p.EIF_cond_exp_isfa_ista. I performed
c.get_parameter_names()
['tau_refrac', 'a', 'tau_m', 'e_rev_E', 'i_offset', 'cm', 'delta_T', 'e_rev_I', 'v_thresh', 'b', 'tau_syn_E', 'v_reset', 'v_spike', 'tau_syn_I', 'tau_w', 'v_rest']
Which gave the above result. I tried to do this
getattr(c,'cm')
Traceback (most recent call last):
File "<pyshell#55>", line 1, in <module>
getattr(c,'cm')
AttributeError: type object 'EIF_cond_exp_isfa_ista' has no attribute 'cm'
I see that there is a parameter called 'cm' but getattr(c,'cm') is throwing an error. I think I'm missing something here.
My code can run,but when I debug,it can enter the Subroutine.And the error is:"decoding Unicode is not supported".
I use anaconda.When I open the untitled0.py,the encoding is UTF-8 at the bottom of the screen,but when I open the fhmm_exact.py,the encoding is UTF-8-GUESSED.
Traceback (most recent call last):
File "<ipython-input-1-f6910c2dfa77>", line 1, in <module>
debugfile('/home/wenwu/untitled0.py', wdir='/home/wenwu')
File "/home/wenwu/anaconda/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 702, in debugfile
debugger.run("runfile(%r, args=%r, wdir=%r)" % (filename, args, wdir))
File "/home/wenwu/anaconda/lib/python2.7/bdb.py", line 400, in run
exec cmd in globals, locals
File "<string>", line 1, in <module>
File "/home/wenwu/anaconda/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 682, in runfile
execfile(filename, namespace)
File "/home/wenwu/anaconda/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 78, in execfile
builtins.execfile(filename, *where)
File "/home/wenwu/untitled0.py", line 37, in <module>
fhmm.disaggregate(test_elec.mains(),output,sample_period = 60)
File "/home/wenwu/nilmtk/nilmtk/disaggregate/fhmm_exact.py", line 287, in disaggregate
mains_data_location = '{}/elec/meter1'.format(building_path)
File "/home/wenwu/nilmtk/nilmtk/disaggregate/fhmm_exact.py", line 287, in disaggregate
mains_data_location = '{}/elec/meter1'.format(building_path)
File "/home/wenwu/anaconda/lib/python2.7/bdb.py", line 49, in trace_dispatch
return self.dispatch_line(frame)
File "/home/wenwu/anaconda/lib/python2.7/bdb.py", line 67, in dispatch_line
self.user_line(frame)
File "/home/wenwu/anaconda/lib/python2.7/pdb.py", line 158, in user_line
self.interaction(frame, None)
File "/home/wenwu/anaconda/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 488, in interaction
self.notify_spyder(frame) #-----Spyder-specific-------------------------
File "/home/wenwu/anaconda/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 432, in notify_spyder
fname = unicode(fname, "utf-8")
TypeError: decoding Unicode is not supported
The following is code.
untitled0.py*
from matplotlib import rcParams
import matplotlib.pyplot as plt
rcParams['figure.figsize'] = (13,6)
plt.style.use('ggplot')
from nilmtk import DataSet,TimeFrame,MeterGroup,HDFDataStore
train = DataSet('/home/wenwu/redd.h5')
test = DataSet('/home/wenwu/redd.h5')
building = 1
train.set_window(end = '30-4-2011')
test.set_window(start = '30-4-2011')
train_elec = train.buildings[1].elec
test_elec = test.buildings[1].elec
fridge_meter = train_elec['fridge']
fridge_df = fridge_meter.load().next()
fridge_df.head()
mains = train_elec.mains()
mains_df = mains.load().next()
top_5_train_elec = train_elec.submeters().select_top_k(k = 5)
from nilmtk.disaggregate import fhmm_exact
from nilmtk.metrics import f1_score
fhmm = fhmm_exact.FHMM()
fhmm.train(top_5_train_elec,sample_period = 60)
disag_filename = '/home/wenwu/redd-disag-fhmm.h5'
output = HDFDataStore(disag_filename,'w')
fhmm.disaggregate(test_elec.mains(),output,sample_period = 60)
output.close()
disag_fhmm = DataSet(disag_filename)
disag_fhmm_elec = disag_fhmm.buildings[building].elec
f1_fhmm = f1_score(disag_fhmm_elec,test_elec)
f1_fhmm.plot(kind = 'barh')
disaggreate part of fhmm_exact.py
def disaggregate(self, mains, output_datastore, **load_kwargs):
'''Disaggregate mains according to the model learnt previously.
Parameters
----------
mains : nilmtk.ElecMeter or nilmtk.MeterGroup
output_datastore : instance of nilmtk.DataStore subclass
For storing power predictions from disaggregation algorithm.
output_name : string, optional
The `name` to use in the metadata for the `output_datastore`.
e.g. some sort of name for this experiment. Defaults to
"NILMTK_FHMM_<date>"
resample_seconds : number, optional
The desired sample period in seconds.
**load_kwargs : key word arguments
Passed to `mains.power_series(**kwargs)`
'''
import warnings
warnings.filterwarnings("ignore", category=Warning)
MIN_CHUNK_LENGTH = 100
if not self.model:
raise RuntimeError(
"The model needs to be instantiated before"
" calling `disaggregate`. For example, the"
" model can be instantiated by running `train`.")
# Extract optional parameters from load_kwargs
date_now = datetime.now().isoformat().split('.')[0]
output_name = load_kwargs.pop('output_name', 'NILMTK_FHMM_' + date_now)
resample_seconds = load_kwargs.pop('resample_seconds', 60)
resample_rule = '{:d}S'.format(resample_seconds)
timeframes = []
building_path = '/building{}'.format(mains.building())
mains_data_location = '{}/elec/meter1'.format(building_path)
data_is_available = False
for chunk in mains.power_series(**load_kwargs):
# Check that chunk is sensible size before resampling
if len(chunk) < MIN_CHUNK_LENGTH:
continue
# Record metadata
timeframes.append(chunk.timeframe)
measurement = chunk.name
chunk = chunk.resample(rule=resample_rule)
# Check chunk size *again* after resampling
if len(chunk) < MIN_CHUNK_LENGTH:
continue
# Start disaggregation
predictions = self.disaggregate_chunk(chunk)
for meter in predictions.columns:
data_is_available = True
meter_instance = meter.instance()
cols = pd.MultiIndex.from_tuples([chunk.name])
predicted_power = predictions[[meter]]
output_df = pd.DataFrame(predicted_power)
output_df.columns = pd.MultiIndex.from_tuples([chunk.name])
output_datastore.append('{}/elec/meter{}'
.format(building_path, meter_instance),
output_df)
# Copy mains data to disag output
output_datastore.append(key=mains_data_location,
value=pd.DataFrame(chunk, columns=cols))
if not data_is_available:
return
##################################
# Add metadata to output_datastore
# TODO: `preprocessing_applied` for all meters
# TODO: split this metadata code into a separate function
# TODO: submeter measurement should probably be the mains
# measurement we used to train on, not the mains measurement.
# DataSet and MeterDevice metadata:
meter_devices = {
'FHMM': {
'model': 'FHMM',
'sample_period': resample_seconds,
'max_sample_period': resample_seconds,
'measurements': [{
'physical_quantity': measurement[0],
'type': measurement[1]
}]
},
'mains': {
'model': 'mains',
'sample_period': resample_seconds,
'max_sample_period': resample_seconds,
'measurements': [{
'physical_quantity': measurement[0],
'type': measurement[1]
}]
}
}
merged_timeframes = merge_timeframes(timeframes, gap=resample_seconds)
total_timeframe = TimeFrame(merged_timeframes[0].start,
merged_timeframes[-1].end)
dataset_metadata = {'name': output_name, 'date': date_now,
'meter_devices': meter_devices,
'timeframe': total_timeframe.to_dict()}
output_datastore.save_metadata('/', dataset_metadata)
# Building metadata
# Mains meter:
elec_meters = {
1: {
'device_model': 'mains',
'site_meter': True,
'data_location': mains_data_location,
'preprocessing_applied': {}, # TODO
'statistics': {
'timeframe': total_timeframe.to_dict()
}
}
}
# TODO: FIX THIS! Ugly hack for now
# Appliances and submeters:
appliances = []
for i, meter in enumerate(self.meters):
meter_instance = meter.instance()
for app in meter.appliances:
appliance = {
'meters': [meter_instance],
'type': app.identifier.type,
'instance': app.identifier.instance
# TODO this `instance` will only be correct when the
# model is trained on the same house as it is tested on.
# https://github.com/nilmtk/nilmtk/issues/194
}
appliances.append(appliance)
elec_meters.update({
meter_instance: {
'device_model': 'FHMM',
'submeter_of': 1,
'data_location': ('{}/elec/meter{}'
.format(building_path, meter_instance)),
'preprocessing_applied': {}, # TODO
'statistics': {
'timeframe': total_timeframe.to_dict()
}
}
})
# Setting the name if it exists
if meter.name:
if len(meter.name) > 0:
elec_meters[meter_instance]['name'] = meter.name
building_metadata = {
'instance': mains.building(),
'elec_meters': elec_meters,
'appliances': appliances
}
output_datastore.save_metadata(building_path, building_metadata)