For the problem formulation
import pyomo.environ as pe
model = pe.AbstractModel()
model.I = pe.Set()
model.p = model.Param(model.I)
model.create_instance("input.dat")
and the input.dat
set I := 1 2 3 ;
param p :=
1 0.1
2 0.2
3 0.3
;
param q :=
1 1.1
2 2.2
3 3.3
;
The following error is shown
AttributeError: 'AbstractModel' object has no attribute 'q'
How to silence create_instance in this case? The model is fully specified. The "excess" data (parameter q in this case) is needed for another model and the models share this input.dat. I could go with a try/except for the AttributeError and just carry on I guess, but then I would need to guard each create_instance call. I looked for a "skip_undefined" kwarg or similar in the documentation. Is there another preferred way to handle this situation?
According to the documentation, if you load your data using the method load from the class DataPortal, the parameters not used by the model are omitted.
Therefore you may try:
from pyomo.environ import *
data = DataPortal()
model = AbstractModel()
data.load(filename='./input.dat')
model.I = Set()
model.p = model.Param(model.I)
instance = model.create_instance(data)
Related
I have a DataStream <pyflink.datastream.data_stream.DataStream> coming from a CoFlatMapFunction (simplified here):
%flink.pyflink
# join two streams and update the rule-set
class MyCoFlatMapFunction(CoFlatMapFunction):
def open(self, runtime_context: RuntimeContext):
state_desc = MapStateDescriptor('map', Types.STRING(), Types.BOOLEAN())
self.state = runtime_context.get_map_state(state_desc)
def bool_from_user_number(self, user_number: int):
'''Retunrs True if user_number is greater than 0, False otherwise.'''
if user_number > 0:
return True
else:
return False
def flat_map1(self, value):
'''This method is called for each element in the first of the connected streams'''
self.state.put(value[1], self.bool_from_user_number(value[2]))
def flat_map2(self, value):
'''This method is called for each element in the second of the connected streams (exchange_server_tickers_data_py)'''
current_dateTime = datetime.now()
dt = current_dateTime
x = value[1]
y = value[2]
yield Row(dt, x, y)
def generate__ds(st_env):
# interpret the updating Tables as DataStreams
type_info1 = Types.ROW([Types.SQL_TIMESTAMP(), Types.STRING(), Types.INT()])
ds1 = st_env.to_append_stream(table_1 , type_info=type_info1)
type_info2 = Types.ROW([Types.SQL_TIMESTAMP(), Types.STRING(), Types.STRING()])
ds2 = st_env.to_append_stream(table_2 , type_info=type_info2)
output_type_info = Types.ROW([ Types.PICKLED_BYTE_ARRAY() ,Types.STRING(),Types.STRING() ])
# Connect the two streams
connected_ds = ds1.connect(ds2)
# Apply the CoFlatMapFunction
ds = connected_ds.key_by(lambda a: a[0], lambda a: a[0]).flat_map(MyCoFlatMapFunction(), output_type_info)
return ds
ds = generate__ds(st_env)
The output, however, I am unable to view, either via registering it as a view / table, writing to a sink table or (the best case) using a Kinesis Streams sink to write data from the Flink stream into a Kinesis stream. Firehouse would also not fit my use case as the 30 second latency would be too long. Any help would be appreciated, thanks!
What I have tried:
Registering it as a view / table like so:
# interpret the DataStream as a Table
input_table = st_env.from_data_stream(ds).alias("dt", "x", "y")
z.show(input_table, stream_type="update")
Which gives an error of:
Query schema: [dt: RAW('[B', '...'), x: STRING, y: STRING]
Sink schema: [dt: RAW('[B', ?), x: STRING, y: STRING]
I have also tried writing to a sink table, like so:
%flink.pyflink
# create a sink table to emit results
st_env.execute_sql("""DROP TABLE IF EXISTS table_sink""")
st_env.execute_sql("""
CREATE TABLE table_sink (
dt RAW('[B', '...'),
x VARCHAR(32),
y STRING
) WITH (
'connector' = 'print'
)
""")
# convert the Table API table to a SQL view
table = st_env.from_data_stream(ds).alias("dt", "spread", "spread_orderbook")
st_env.execute_sql("""DROP TEMPORARY VIEW IF EXISTS table_api_table""")
st_env.create_temporary_view('table_api_table', table)
# emit the Table API table
st_env.execute_sql("INSERT INTO table_sink SELECT * FROM table_api_table").wait()
I get the error:
org.apache.flink.table.api.ValidationException: Unable to restore the RAW type of class '[B' with serializer snapshot '...'.
I have also tried to use a sink and add_sink to write the data to a sink, which would be an AWS kinesis data stream like in these Docs, like so:
%flink.pyflink
from pyflink.common.serialization import JsonRowSerializationSchema
from pyflink.datastream.connectors import KinesisStreamsSink
output_type_info = Types.ROW([Types.SQL_TIMESTAMP(), Types.STRING(), Types.STRING()])
serialization_schema = JsonRowSerializationSchema.Builder().with_type_info(output_type_info).build()
# Required
sink_properties = {
'aws.region': 'eu-west-2'
}
kds_sink = KinesisStreamsSink.builder()
.set_kinesis_client_properties(sink_properties)
.set_serialization_schema(SimpleStringSchema())
.set_partition_key_generator(PartitionKeyGenerator
.fixed())
.set_stream_name("test_stream")
.set_fail_on_error(False)
.set_max_batch_size(500)
.set_max_in_flight_requests(50)
.set_max_buffered_requests(10000)
.set_max_batch_size_in_bytes(5 * 1024 * 1024)
.set_max_time_in_buffer_ms(5000)
.set_max_record_size_in_bytes(1 * 1024 * 1024)
.build()
ds.sink_to(kds_sink)
Which i assume would work, but KinesisStreamsSink is not found in pyflink.datastream.connectors and I am unable to find any documentation on how to do this within AWS Kinesis Analytics Studio. Any help would be much much appreciated, thank you! How would I go about writing the data to a Kinesis Streams sink / converting it to a table?
Okay, i have figured it out. There were a couple issues with the particular Pyflink version available on AWS Kinesis Analytics Studio (1.13). The error messages themselves were not that useful, so for anyone who is having issues themselves I would really recommend viewing the errors in the Flink Web UI. Firstly, the MapStateDescriptor datatypes must be specified using Types.PICKLED_BYTE_ARRAY(). Secondly, not shown in the Qn, but each MapStateDescriptor must have a distinct name. I also found that using Row from pyflink.common threw errors for me. It worked better for me to switch to using use Tuples by specifying Types.TUPLE() as is done in this example. I also had to switch to specifying the output as a tuple.
Another thing I have not done is specify a watermark strategy for the DataStream, which could potentially be done by extracting the timestamp from the first field, and assign watermarks based on knowledge of the stream:
class MyTimestampAssigner(TimestampAssigner):
def extract_timestamp(self, value, record_timestamp: int) -> int:
return int(value[0])
watermark_strategy = WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(5)).with_timestamp_assigner(MyTimestampAssigner())
ds = ds.assign_timestamps_and_watermarks(watermark_strategy)
# the first field has been used for timestamp extraction, and is no longer necessary
# replace first field with a logical event time attribute
table = st_env.from_data_stream(ds, col("dt").rowtime, col('f0'), col('f1'))
But i have instead created a sink table for writing to a Kinesis Data Stream again as an output. In total, the corrected code would look something like this:
from pyflink.table.expressions import col
from pyflink.datastream.state import MapStateDescriptor
from pyflink.datastream.functions import RuntimeContext, CoFlatMapFunction
from pyflink.common.typeinfo import Types
from pyflink.common import Duration as Time, WatermarkStrategy, Duration
from pyflink.common.typeinfo import Types
from pyflink.common.watermark_strategy import TimestampAssigner
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import KeyedProcessFunction, RuntimeContext
from pyflink.datastream.state import ValueStateDescriptor
from datetime import datetime
# Register the tables in the env
table1 = st_env.from_path("sql_table_1")
table2 = st_env.from_path("sql_table_2")
# interpret the updating Tables as DataStreams
type_info1 = Types.TUPLE([Types.SQL_TIMESTAMP(), Types.STRING(), Types.INT()])
ds1 = st_env.to_append_stream(table2, type_info=type_info1)
type_info2 = Types.TUPLE([Types.SQL_TIMESTAMP(), Types.STRING(), Types.STRING()])
ds2 = st_env.to_append_stream(table1, type_info=type_info2)
# join two streams and update the rule-set state
class MyCoFlatMapFunction(CoFlatMapFunction):
def open(self, runtime_context: RuntimeContext):
'''This method is called when the function is opened in the runtime. It is the initialization purposes.'''
# Map state that we use to maintain the filtering and rules
state_desc = MapStateDescriptor('map', Types.PICKLED_BYTE_ARRAY(), Types.PICKLED_BYTE_ARRAY())
self.state = runtime_context.get_map_state(state_desc)
# maintain state 2
ob_state_desc = MapStateDescriptor('map_OB', Types.PICKLED_BYTE_ARRAY(), Types.PICKLED_BYTE_ARRAY())
self.ob_state = runtime_context.get_map_state(ob_state_desc)
# called on ds1
def flat_map1(self, value):
'''This method is called for each element in the first of the connected streams '''
list_res = value[1].split('|')
for i in list_res:
time = datetime.utcnow().replace(microsecond=0)
yield (time, f"{i}_one")
# called on ds2
def flat_map2(self, value):
'''This method is called for each element in the second of the connected streams'''
list_res = value[1].split('|')
for i in list_res:
time = datetime.utcnow().replace(microsecond=0)
yield (time, f"{i}_two")
connectedStreams = ds1.connect(ds2)
output_type_info = Types.TUPLE([Types.SQL_TIMESTAMP(), Types.STRING()])
ds = connectedStreams.key_by(lambda value: value[1], lambda value: value[1]).flat_map(MyCoFlatMapFunction(), output_type=output_type_info)
name = 'output_table'
ds_table_name = 'temporary_table_dump'
st_env.execute_sql(f"""DROP TABLE IF EXISTS {name}""")
def create_table(table_name, stream_name, region, stream_initpos):
return """ CREATE TABLE {0} (
f0 TIMESTAMP(3),
f1 STRING,
WATERMARK FOR f0 AS f0 - INTERVAL '5' SECOND
)
WITH (
'connector' = 'kinesis',
'stream' = '{1}',
'aws.region' = '{2}',
'scan.stream.initpos' = '{3}',
'sink.partitioner-field-delimiter' = ';',
'sink.producer.collection-max-count' = '100',
'format' = 'json',
'json.timestamp-format.standard' = 'ISO-8601'
) """.format(
table_name, stream_name, region, stream_initpos
)
# Creates a sink table writing to a Kinesis Data Stream
st_env.execute_sql(create_table(name, 'output-test', 'eu-west-2', 'LATEST'))
table = st_env.from_data_stream(ds)
st_env.execute_sql(f"""DROP TEMPORARY VIEW IF EXISTS {ds_table_name}""")
st_env.create_temporary_view(ds_table_name, table)
# emit the Table API table
st_env.execute_sql(f"INSERT INTO {name} SELECT * FROM {ds_table_name}").wait()
I have a model that I need to solve many times, with different objective function coefficients.
Naturally, I want to spend as little time as possible on updating the model.
My current setup is as follows (simplified):
Abstract model:
def obj_rule(m):
return sum(Dist[m, n] * m.flow[m,n] for (m,n) in m.From * m.To)
m.obj = Objective(rule=obj_rule, sense=minimize)
After solve, I update the concrete model instance mi this way, we new values in Dist:
mi.obj = sum(Dist[m, n] * mi.flow[m,n] for (m,n) in mi.From * mi.To)
However, I see that the update line takes a lot of time - ca. 1/4 of the overall solution time, several seconds for bigger cases.
Is there some faster way of updating the objective function?
(After all, in the usual way of saving an LP model, the objective function coefficients are in a separate vector, so changing them should not affect anything else.)
Do you have a reason to define an Abstract model before creating your Concrete model? If you define your concrete model with the rule you show above, you should be able to just update your data and re-solve the model without a lot of overhead since you do don't redefine the objective object. Here's a simple example where I change the values of the cost parameter and re-solve.
import pyomo.environ as pyo
a = list(range(2)) # set the variables define over
#%% Begin basic model
model = pyo.ConcreteModel()
model.c = pyo.Param(a,initialize={0:5,1:3},mutable=True)
model.x = pyo.Var(a,domain = pyo.Binary)
model.con = pyo.Constraint(expr=model.x[0] + model.x[1] <= 1)
def obj_rule(model):
return(sum(model.x[ind] * model.c[ind] for ind in a))
model.obj = pyo.Objective(rule=obj_rule,sense=pyo.maximize)
#%% Solve the first time
solver = pyo.SolverFactory('glpk')
res=solver.solve(model)
print('x[0]: {} \nx[1]: {}'.format(pyo.value(model.x[0]),
pyo.value(model.x[1])))
# x[0]: 1
# x[1]: 0
#%% Update data and re-solve
model.c.reconstruct({0:0,1:5})
res=solver.solve(model)
print('x[0]: {} \nx[1]: {}'.format(pyo.value(model.x[0]),
pyo.value(model.x[1])))
# x[0]: 0
# x[1]: 1
I have a pyomo ConcreteModel() which I solve repeatedly within another stochastic optimization process whereas one or more parameters are changed on the model.
The basic process can be described as follows:
# model is created as a pyomo.ConcreteModel()
for i in range(0, 10):
# change some parameter on the model
opt = SolverFactory('gurobi', solver_io='lp')
# how can I check here if the changed model/lp-file is valid?
results = opt.solve(model)
Now I get an error for some cases where the model and LP file (see gist) seems to contain NaN values:
ERROR: Solver (gurobi) returned non-zero return code (1)
ERROR: Solver log: Academic license - for non-commercial use only Error
reading LP format file /tmp/tmp8agg07az.pyomo.lp at line 1453 Unrecognized
constraint RHS or sense Neighboring tokens: " <= nan c_u_x1371_: +1 x434
<= nan "
Unable to read file Traceback (most recent call last):
File "<stdin>", line 5, in <module> File
"/home/cord/.anaconda3/lib/python3.6/site-
packages/pyomo/solvers/plugins/solvers/GUROBI_RUN.py", line 61, in
gurobi_run
model = read(model_file)
File "gurobi.pxi", line 2652, in gurobipy.read
(../../src/python/gurobipy.c:127968) File "gurobi.pxi", line 72, in
gurobipy.gurobi.read (../../src/python/gurobipy.c:125753)
gurobipy.GurobiError: Unable to read model Freed default Gurobi
environment
Of course, the first idea would be to prevent setting these NaN-values. But I don't know why they occur anyhow and want to figure out when the model breaks due to a wrong structure caused by NaNs.
I know that I can catch the solver status and termination criterion from the SolverFactory() object. But the error obviously occurs somewhere before the solving process due to the invalid changed values.
How can I can catch these kinds of errors for different solvers before solving i. e. check if the model/lp-file is valid before applying a solver? Is there some method e.g. check_model() which delivers True or False if the model is (not) valid or something similar?
Thanks in advance!
If you know that the error is taking place when the parameter values are being changed, then you could test to see whether the sum of all relevant parameter values is a valid number. After all, NaN + 3 = NaN.
Since you are getting NaN, I am going to guess that you are importing parameter values using Pandas from an Excel spreadsheet? There is a way to convert all the NaNs to a default number.
Code example for parameter check:
>>> from pyomo.environ import *
>>> m = ConcreteModel()
>>> m.p1 = Param(initialize=1)
>>> m.p2 = Param(initialize=2)
>>> for p in m.component_data_objects(ctype=Param):
... print(p.name)
...
p1
p2
>>> import numpy
>>> m.p3 = Param(initialize=numpy.nan)
>>> import math
>>> math.isnan(value(sum(m.component_data_objects(ctype=Param))))
True
Indexed, Mutable Parameters:
>>> from pyomo.environ import *
>>> m = ConcreteModel()
>>> m.i = RangeSet(2)
>>> m.p = Param(m.i, initialize={1: 1, 2:2}, mutable=True)
>>> import math
>>> import numpy
>>> math.isnan(value(sum(m.component_data_objects(ctype=Param))))
False
>>> m.p[1] = numpy.nan
>>> math.isnan(value(sum(m.component_data_objects(ctype=Param))))
True
I'm trying to create time series from a netCDF file (accessed via Thredds server) with python. The code I use seems correct, but the values of the variable amb reading are 'masked'. I'm new into python and I'm not familiar with the formats. Any idea of how can I read the data?
This is the code I use:
import netCDF4
import pandas as pd
import datetime as dt
import matplotlib.pyplot as plt
from datetime import datetime, timedelta #
dayFile = datetime.now() - timedelta(days=1)
dayFile = dayFile.strftime("%Y%m%d")
url='http://nomads.ncep.noaa.gov:9090/dods/nam/nam%s/nam1hr_00z' %(dayFile)
# NetCDF4-Python can open OPeNDAP dataset just like a local NetCDF file
nc = netCDF4.Dataset(url)
varsInFile = nc.variables.keys()
lat = nc.variables['lat'][:]
lon = nc.variables['lon'][:]
time_var = nc.variables['time']
dtime = netCDF4.num2date(time_var[:],time_var.units)
first = netCDF4.num2date(time_var[0],time_var.units)
last = netCDF4.num2date(time_var[-1],time_var.units)
print first.strftime('%Y-%b-%d %H:%M')
print last.strftime('%Y-%b-%d %H:%M')
# determine what longitude convention is being used
print lon.min(),lon.max()
# Specify desired station time series location
# note we add 360 because of the lon convention in this dataset
#lati = 36.605; loni = -121.85899 + 360. # west of Pacific Grove, CA
lati = 41.4; loni = -100.8 +360.0 # Georges Bank
# Function to find index to nearest point
def near(array,value):
idx=(abs(array-value)).argmin()
return idx
# Find nearest point to desired location (no interpolation)
ix = near(lon, loni)
iy = near(lat, lati)
print ix,iy
# Extract desired times.
# 1. Select -+some days around the current time:
start = netCDF4.num2date(time_var[0],time_var.units)
stop = netCDF4.num2date(time_var[-1],time_var.units)
time_var = nc.variables['time']
datetime = netCDF4.num2date(time_var[:],time_var.units)
istart = netCDF4.date2index(start,time_var,select='nearest')
istop = netCDF4.date2index(stop,time_var,select='nearest')
print istart,istop
# Get all time records of variable [vname] at indices [iy,ix]
vname = 'dswrfsfc'
var = nc.variables[vname]
hs = var[istart:istop,iy,ix]
tim = dtime[istart:istop]
# Create Pandas time series object
ts = pd.Series(hs,index=tim,name=vname)
The var data are not read as I expected, apparently because data is masked:
>>> hs
masked_array(data = [-- -- -- ..., -- -- --],
mask = [ True True True ..., True True True],
fill_value = 9.999e+20)
The var name, and the time series are correct, as well of the rest of the script. The only thing that doesn't work is the var data retrieved. This is the time serie I get:
>>> ts
2016-10-25 00:00:00.000000 NaN
2016-10-25 01:00:00.000000 NaN
2016-10-25 02:00:00.000006 NaN
2016-10-25 03:00:00.000000 NaN
2016-10-25 04:00:00.000000 NaN
... ... ... ... ...
2016-10-26 10:00:00.000000 NaN
2016-10-26 11:00:00.000006 NaN
Name: dswrfsfc, dtype: float32
Any help will be appreciated!
Hmm, this code looks familiar. ;-)
You are getting NaNs because the NAM model you are trying to access now uses longitude in the range [-180, 180] instead of the range [0, 360]. So if you request loni = -100.8 instead of loni = -100.8 +360.0, I believe your code will return non-NaN values.
It's worth noting, however, that the task of extracting time series from multidimensional gridded data is now much easier with xarray, because you can simply select a dataset closest to a lon,lat point and then plot any variable. The data only gets loaded when you need it, not when you extract the dataset object. So basically you now only need:
import xarray as xr
ds = xr.open_dataset(url) # NetCDF or OPeNDAP URL
lati = 41.4; loni = -100.8 # Georges Bank
# Extract a dataset closest to specified point
dsloc = ds.sel(lon=loni, lat=lati, method='nearest')
# select a variable to plot
dsloc['dswrfsfc'].plot()
Full notebook here: http://nbviewer.jupyter.org/gist/rsignell-usgs/d55b37c6253f27c53ef0731b610b81b4
I checked your approach with xarray. Works great to extract Solar radiation data! I can add that the first point is not defined (NaN) because the model starts calculating there, so there is no accumulated radiation data (to calculate hourly global radiation). So that is why it is masked.
Something everyone overlooked is that the output is not correct. It does look ok (at noon= sunshine, at nmidnight=0, dark), but the daylength is not correct! I checked it for 52 latitude north and 5.6 longitude (east) (November) and daylength is at least 2 hours too much! (The NOAA Panoply viewer for Netcdf databases gives similar results)
When I use extra in a certain way on a Django queryset (call it qs), the result of qs.count() is different than len(qs.all()). To reproduce:
Make an empty Django project and app, then add a trivial model:
class Baz(models.Model):
pass
Now make a few objects:
>>> Baz(id=1).save()
>>> Baz(id=2).save()
>>> Baz(id=3).save()
>>> Baz(id=4).save()
Using the extra method to select only some of them produces the expected count:
>>> Baz.objects.extra(where=['id > 2']).count()
2
>>> Baz.objects.extra(where=['-id < -2']).count()
2
But add a select clause to the extra and refer to it in the where clause, and the count is suddenly wrong, even though the result of all() is correct:
>>> Baz.objects.extra(select={'negid': '0 - id'}, where=['"negid" < -2']).all()
[<Baz: Baz object>, <Baz: Baz object>] # As expected
>>> Baz.objects.extra(select={'negid': '0 - id'}, where=['"negid" < -2']).count()
0 # Should be 2
I think the problem has to do with django.db.models.sql.query.BaseQuery.get_count(). It checks whether the BaseQuery's select or aggregate_select attributes have been set; if so, it uses a subquery. But django.db.models.sql.query.BaseQuery.add_extra adds only to the BaseQuery's extra attribute, not select or aggregate_select.
How can I fix the problem? I know I could just use len(qs.all()), but it would be nice to be able to pass the extra'ed queryset to other parts of the code, and those parts may call count() without knowing that it's broken.
Redefining get_count() and monkeypatching appears to fix the problem:
def get_count(self):
"""
Performs a COUNT() query using the current filter constraints.
"""
obj = self.clone()
if len(self.select) > 1 or self.aggregate_select or self.extra:
# If a select clause exists, then the query has already started to
# specify the columns that are to be returned.
# In this case, we need to use a subquery to evaluate the count.
from django.db.models.sql.subqueries import AggregateQuery
subquery = obj
subquery.clear_ordering(True)
subquery.clear_limits()
obj = AggregateQuery(obj.model, obj.connection)
obj.add_subquery(subquery)
obj.add_count_column()
number = obj.get_aggregation()[None]
# Apply offset and limit constraints manually, since using LIMIT/OFFSET
# in SQL (in variants that provide them) doesn't change the COUNT
# output.
number = max(0, number - self.low_mark)
if self.high_mark is not None:
number = min(number, self.high_mark - self.low_mark)
return number
django.db.models.sql.query.BaseQuery.get_count = quuux.get_count
Testing:
>>> Baz.objects.extra(select={'negid': '0 - id'}, where=['"negid" < -2']).count()
2
Updated to work with Django 1.2.1:
def basequery_get_count(self, using):
"""
Performs a COUNT() query using the current filter constraints.
"""
obj = self.clone()
if len(self.select) > 1 or self.aggregate_select or self.extra:
# If a select clause exists, then the query has already started to
# specify the columns that are to be returned.
# In this case, we need to use a subquery to evaluate the count.
from django.db.models.sql.subqueries import AggregateQuery
subquery = obj
subquery.clear_ordering(True)
subquery.clear_limits()
obj = AggregateQuery(obj.model)
obj.add_subquery(subquery, using=using)
obj.add_count_column()
number = obj.get_aggregation(using=using)[None]
# Apply offset and limit constraints manually, since using LIMIT/OFFSET
# in SQL (in variants that provide them) doesn't change the COUNT
# output.
number = max(0, number - self.low_mark)
if self.high_mark is not None:
number = min(number, self.high_mark - self.low_mark)
return number
models.sql.query.Query.get_count = basequery_get_count
I'm not sure if this fix will have other unintended consequences, however.