Given the following code based on Pyomo docs (https://pyomo.readthedocs.io/en/stable/developer_reference/expressions/managing.html#identifying-components-and-variables and https://pyomo.readthedocs.io/en/stable/pyomo_modeling_components/Expressions.html#expression-objects) I would expect to find a non-empty list, i.e. a list containing the variable model.x. Where is my misconception here?
import pyomo.environ as pyo
from pyomo.core.expr.current import identify_variables
model = pyo.ConcreteModel()
model.x = pyo.Var(initialize=1.0)
def _e(m,i):
return m.x*i
model.e = pyo.Expression([1,2,3], rule=_e)
assert len(list(identify_variables(model.e, include_fixed=True))) > 0
you need to convert the expression model.e into a list of expressions by using the values() method. Then, you can pass the list of expressions to the identify_variables function.
import pyomo.environ as pyo
from pyomo.core.expr.current import identify_variables
model = pyo.ConcreteModel()
model.x = pyo.Var(initialize=1.0)
def _e(m,i):
return m.x*i
model.e = pyo.Expression([1,2,3], rule=_e)
# Convert expression to list of expressions
expr_list = list(model.e.values())
# Pass list of expressions to identify_variables function
assert len(list(identify_variables(expr_list, include_fixed=True))) > 0
For reference, this is what I ended up with to have a non-empty list if any expression uses a Var:
from itertools import chain
import pyomo.environ as pyo
from pyomo.core.expr.current import identify_variables
model = pyo.ConcreteModel()
model.x = pyo.Var(initialize=1.0)
def _e(m,i):
return m.x*i
model.e = pyo.Expression([1,2,3], rule=_e)
assert len(list(chain(*(identify_variables(value) for value in list(component.values()))))) > 0
Related
I am working on a small optimization model with some disjunctions. The way I did in a concrete model worked well:
from pyomo.environ import *
m = ConcreteModel()
m.d1 = Disjunct()
m.d2 = Disjunct()
m.d1.sub1 = Disjunct()
m.d1.sub2 = Disjunct()
m.d1.disj = Disjunction(expr=[m.d1.sub1, m.d1.sub2])
m.disj = Disjunction(expr=[m.d1, m.d2])
But now I tranfered the concrete model into an abstract formulation. I was able to fix everything instead of nesting the disjunctions. The way I did it was like:
#Disjunct 1
def _op_mode1(self, op_mode, t):
m = op_mode.model()
op_mode.c1 = po.Constraint(expr=m.x[t] == True)
#Disjunct 2
def _op_mode2(self, op_mode, t):
m = op_mode.model()
op_mode.c1 = po.Constraint(expr=m.x[t] == False)
#Disjunction 1
def _op_modes(self,m, t):
return [m.mode1[t], m.mode2[t]]
#Adding Components
self.model.del_component("mode1")
self.model.del_component("mode1_index")
self.model.add_component("mode1", pogdp.Disjunct(self.model.T, rule=self._op_mode1))
self.model.del_component("mode2")
self.model.del_component("mode2_index")
self.model.add_component("mode2", pogdp.Disjunct(self.model.T, rule=self._op_mode1))
self.model.del_component("modes")
self.model.del_component("modes_index")
self.model.add_component("modes", pogdp.Disjunction(self.model.T, rule=self._op_modes))`
As I previously mentioned, this works fine. But I haven`t found any way to nest the disjunctions. Pyomo alsways complains about the second layer of the disjuncts like "sub1".
Would anybody could give me a hint?
Many greetings
Joerg
The issue with the latest model above is that you are declaring m.d1 and m.d2 for each element of m.T, but they overwrite each other each time since they have the same name. You should be seeing warning messages logged for this. So if you uncomment your pprint of the model, you'll see that you only have the last ones you declared (with constraints on x[10]). So the first 9 Disjunctions in m.disjunction_ are disjunctions of Disjuncts that do not exist. The simplest fix for this is to give the disjuncts unique names when you declare them:
import pyomo.environ as pyo
import pyomo.gdp as pogdp
model = pyo.ConcreteModel()
model.T = pyo.RangeSet(0, 10)
model.x=pyo.Var(model.T,bounds=(-2, 10))
model.y=pyo.Var(model.T,bounds=(20, 30))
# This was also a duplicate declaration:
#model.disjunction_ = pogdp.Disjunction(model.T)
def d1(m, t):
disj = pogdp.Disjunct()
disj.c1= pyo.Constraint(expr=m.x[t] <= 10)
m.add_component('d1_%s' % t, disj)
return disj
def d2(m, t):
disj = pogdp.Disjunct()
disj.c1= pyo.Constraint(expr=m.x[t] >= 10)
m.add_component('d2_%s' % t, disj)
return disj
# sum x,y
def obj_rule(m):
return pyo.quicksum(pyo.quicksum([m.x[t] + m.y[t]], linear=False) for t in
m.T)
model.obj = pyo.Objective(rule=obj_rule)
def _op_mode_test(m, t):
disj1 = d1(m, t)
disj2 = d2(m, t)
return [disj1, disj2]
model.disjunction_ = pogdp.Disjunction(model.T, rule=_op_mode_test)
However, it would be cleaner (and probably easier down the line) to index the Disjuncts by m.T as well, since that's basically what the unique names are doing.
Block (and hence Disjunct rules) are passed the block (or disjunct) to be populated as the first argument. So, an "abstract" equivalent too your concrete model might look something like this:
model = AbstractModel()
#model.Disjunct()
def d1(d):
# populate the `d` disjunct (i.e., `model.d1`) here
pass
#model.Disjunct()
def d2(d):
#d.Disjunct()
def sub1(sd):
# populate the 'sub1' disjunct here
pass
#d.Disjunct()
def sub2(sd):
# populate the 'sub2' disjunct here
pass
d.disj = Disjunction(expr=[d.sub1, d.sub2])
model.disj = Disjunction(expr=[model.d1, model.d2])
There is a more fundamental question as to why you are converting your model over to "abstract" form. Pyomo Abstract models were mostly devised to be familiar to people coming from modeling in AMPL. While they will work with block-structured models, as AMPL was never really designed with blocks in mind, similarly block-oriented Abstract models tend to be unnecessarily cumbersome.
Here ist our new model:
import pyomo.environ as pyo
import pyomo.gdp as pogdp
model = pyo.ConcreteModel()
model.T = pyo.RangeSet(0,10)
model.x=pyo.Var(model.T,bounds=(-2, 10))
model.y=pyo.Var(model.T,bounds=(20, 30))
model.disjunction_=pogdp.Disjunction(model.T)
def d1(m,t):
m.d1 = pogdp.Disjunct()
m.d1.c1= pyo.Constraint(expr=m.x[t] <=10)
def d2(m,t):
m.d2 = pogdp.Disjunct()
m.d2.c1= pyo.Constraint(expr=m.x[t] >=10)
# sum x,y
def obj_rule(m):
return pyo.quicksum(pyo.quicksum([m.x[t] + m.y[t]], linear=False) for t in m.T)
model.obj = pyo.Objective(rule=obj_rule)
def _op_mode_test(m,t):
d1(m,t)
d2(m,t)
return [m.d1,m.d2]
model.disjunction_=pogdp.Disjunction(model.T,rule=_op_mode_test)
#model.pprint()
pyo.TransformationFactory('gdp.bigm').apply_to(model)
solver = pyo.SolverFactory('baron')
solver.solve(model)
print(pyo.value(model.obj))
I think it has something to do with the RangeSet. For a single step it works, but with more than one steps it throws an error: AttributeError: 'NoneType' object has no attribute 'component'
It would be great if you could have a look on it.
Many thanks
Here is the code which works pretty fine with bigm, but not with mbigm or hull transformation:
import pyomo.environ as pyo
import pyomo.gdp as pogdp
model = pyo.ConcreteModel()
model.T = pyo.RangeSet(2)
model.x=pyo.Var(model.T,bounds=(1, 10))
model.y=pyo.Var(model.T,bounds=(1, 100))
def _op_mode_sub(m, t):
m.disj1[t].sub1 = pogdp.Disjunct()
m.disj1[t].sub1.c1= pyo.Constraint(expr=m.y[t] == 60)
m.disj1[t].sub2 = pogdp.Disjunct()
m.disj1[t].sub2.c1= pyo.Constraint(expr=m.y[t] == 100)
return [m.disj1[t].sub1, m.disj1[t].sub2]
def _op_mode(m, t):
m.disj2[t].c1= pyo.Constraint(expr=m.y[t] >= 3)
m.disj2[t].c2= pyo.Constraint(expr=m.y[t] <= 5)
return [m.disj1[t], m.disj2[t]]
model.disj1 = pogdp.Disjunct(model.T)
model.disj2 = pogdp.Disjunct(model.T)
model.disjunction1sub = pogdp.Disjunction(model.T, rule=_op_mode_sub)
model.disjunction1 = pogdp.Disjunction(model.T, rule=_op_mode)
def obj_rule(m, t):
return pyo.quicksum(pyo.quicksum([m.x[t] + m.y[t]], linear=False) for t in m.T)
model.obj = pyo.Objective(rule=obj_rule)
model.pprint()
gdp_relax=pyo.TransformationFactory('gdp.bigm')
gdp_relax.apply_to(model)
solver = pyo.SolverFactory('glpk')
solver.solve(model)
print(pyo.value(model.obj))
I create a model in django
I want to set monthly_wage as monthly_wage=working_days*daily_wage in the annotation
from django.db.models import F
class AnnotationManager(models.Manager):
def __init__(self, **kwargs):
super().__init__()
self.annotations = kwargs
def get_queryset(self):
return super().get_queryset().annotate(**self.annotations)
class DetailsList(models.Model):
month_list=models.ForeignKey('MonthList',on_delete=models.CASCADE,verbose_name='لیست ماهانه ')
worker=models.ForeignKey('Workers',on_delete=models.CASCADE,verbose_name=' نام کارگر')
working_days=models.SmallIntegerField('تعداد روز کارکرد')
daily_wage=models.BigIntegerField('دستمزد روزانه')
advantage=models.BigIntegerField('مزایای ماهانه')
_monthly_wage=0
objects = AnnotationManager(
monthly_wage=F('working_days') * F('daily_wage')
)
but because working_days is smallinteger and daily_wage is biginteger
this error raise:
Expression contains mixed types: SmallIntegerField, BigIntegerField.
You must set output_field.
How can I fix this
You set the output_field, with an ExpressionWrapper [Django-doc] for example:
from django.db.models import BigIntegerField, ExpressionWrapper
class DetailsList(models.Model):
# …
objects = AnnotationManager(
monthly_wage=ExpressionWrapper(
F('working_days') * F('daily_wage'),
output_field=BigIntegerField()
)
)
I am currently coding an Multiple Gradient Descent algorithm, where I use kriged functions.
My problem is that I can't find how to obtain the gradient of the kriged function (I tried to use linearize but I don't know how to make it work).
from __future__ import print_function
from six import moves
from random import shuffle
import sys
import numpy as np
from numpy import linalg as LA
import math
from openmdao.braninkm import F, G, DF, DG
from openmdao.api import Group, Component,IndepVarComp
from openmdao.api import MetaModel
from openmdao.api import KrigingSurrogate, FloatKrigingSurrogate
def rand_lhc(b, k):
# Calculates a random Latin hypercube set of n points in k dimensions within [0,n-1]^k hypercube.
arr = np.zeros((2*b, k))
row = list(moves.xrange(-b, b))
for i in moves.xrange(k):
shuffle(row)
arr[:, i] = row
return arr/b*1.2
class TrigMM(Group):
''' FloatKriging gives responses as floats '''
def __init__(self):
super(TrigMM, self).__init__()
# Create meta_model for f_x as the response
F_mm = self.add("F_mm", MetaModel())
F_mm.add_param('X', val=np.array([0., 0.]))
F_mm.add_output('f_x:float', val=0., surrogate=FloatKrigingSurrogate())
# F_mm.add_output('df_x:float', val=0., surrogate=KrigingSurrogate().linearize)
#F_mm.linearize('X', 'f_x:float')
#F_mm.add_output('g_x:float', val=0., surrogate=FloatKrigingSurrogate())
print('init ok')
self.add('p1', IndepVarComp('X', val=np.array([0., 0.])))
self.connect('p1.X','F_mm.X')
# Create meta_model for f_x as the response
G_mm = self.add("G_mm", MetaModel())
G_mm.add_param('X', val=np.array([0., 0.]))
G_mm.add_output('g_x:float', val=0., surrogate=FloatKrigingSurrogate())
#G_mm.add_output('df_x:float', val=0., surrogate=KrigingSurrogate().linearize)
#G_mm.linearize('X', 'g_x:float')
self.add('p2', IndepVarComp('X', val=np.array([0., 0.])))
self.connect('p2.X','G_mm.X')
from openmdao.api import Problem
prob = Problem()
prob.root = TrigMM()
prob.setup()
u=4
v=3
#training avec latin hypercube
prob['F_mm.train:X'] = rand_lhc(20,2)
prob['G_mm.train:X'] = rand_lhc(20,2)
#prob['F_mm.train:X'] = rand_lhc(10,2)
#prob['G_mm.train:X'] = rand_lhc(10,2)
#prob['F_mm.linearize:X'] = rand_lhc(10,2)
#prob['G_mm.linearize:X'] = rand_lhc(10,2)
datF=[]
datG=[]
datDF=[]
datDG=[]
for i in range(len(prob['F_mm.train:X'])):
datF.append(F(np.array([prob['F_mm.train:X'][i]]),u))
#datG.append(G(np.array([prob['F_mm.train:X'][i]]),v))
data_trainF=np.fromiter(datF,np.float)
for i in range(len(prob['G_mm.train:X'])):
datG.append(G(np.array([prob['G_mm.train:X'][i]]),v))
data_trainG=np.fromiter(datG,np.float)
prob['F_mm.train:f_x:float'] = data_trainF
#prob['F_mm.train:g_x:float'] = data_trainG
prob['G_mm.train:g_x:float'] = data_trainG
Are you going to be writing a Multiple Gradient Descent driver? If so, then OpenMDAO calculates the gradient from a param to an output at the Problem level using the calc_gradient method.
If you take a look at the source code for the pyoptsparse driver:
https://github.com/OpenMDAO/OpenMDAO/blob/master/openmdao/drivers/pyoptsparse_driver.py
The _gradfunc method is a callback function that returns the gradient of the constraints and objectives with respect to the design variables. The Metamodel component has built-in analytic gradients for all (I think) of our surrogates, so you don't even have to declare any there.
If this isn't what you are trying to do, then I may need a little more information about your application.
I am trying to perform feature selection for logistic regression classifier. Originally there are 4 variables: name, location, gender, and label = ethnicity. The three variables, namely the name, give rise to tens of thousands of more "features", for example, name "John Snow" will give rise to 2-letter substrings like 'jo', 'oh', 'hn'... etc. The feature set undergoes DictVectorization.
I am trying to follow this tutorial (http://scikit-learn.org/stable/auto_examples/feature_selection/plot_feature_selection.html) but I am not sure if I am doing it right since the tutorial is using a small number of features while mine has tens of thousands after vectorization. And also the plt.show() shows a blank figure.
# coding=utf-8
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import re
import random
import time
from random import randint
import csv
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from sklearn import svm
from sklearn.metrics import classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import SelectPercentile, f_classif
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_curve
# Assign X and y variables
X = df.raw_name.values
X2 = df.name.values
X3 = df.gender.values
X4 = df.location.values
y = df.ethnicity_scan.values
# Feature extraction functions
def feature_full_name(nameString):
try:
full_name = nameString
if len(full_name) > 1: # not accept name with only 1 character
return full_name
else: return '?'
except: return '?'
def feature_avg_wordLength(nameString):
try:
space = 0
for i in nameString:
if i == ' ':
space += 1
length = float(len(nameString) - space)
name_entity = float(space + 1)
avg = round(float(length/name_entity), 0)
return avg
except:
return 0
def feature_name_entity(nameString2):
space = 0
try:
for i in nameString2:
if i == ' ':
space += 1
return space+1
except: return 0
def feature_gender(genString):
try:
gender = genString
if len(gender) >= 1:
return gender
else: return '?'
except: return '?'
def feature_noNeighborLoc(locString):
try:
x = re.sub(r'^[^, ]*', '', locString) # remove everything before and include first ','
y = x[2:] # remove subsequent ',' and ' '
return y
except: return '?'
def list_to_dict(substring_list):
try:
substring_dict = {}
for i in substring_list:
substring_dict['substring='+str(i)] = True
return substring_dict
except: return '?'
# Transform format of X variables, and spit out a numpy array for all features
my_dict13 = [{'name-entity': feature_name_entity(feature_full_name(i))} for i in X2]
my_dict14 = [{'avg-length': feature_avg_wordLength(feature_full_name(i))} for i in X]
my_dict15 = [{'gender': feature_full_name(i)} for i in X3]
my_dict16 = [{'location': feature_noNeighborLoc(feature_full_name(i))} for i in X4]
my_dict17 = [{'dummy1': 1} for i in X]
my_dict18 = [{'dummy2': random.randint(0,2)} for i in X]
all_dict = []
for i in range(0, len(my_dict)):
temp_dict = dict(my_dict13[i].items() + my_dict14[i].items()
+ my_dict15[i].items() + my_dict16[i].items() + my_dict17[i].items() + my_dict18[i].items()
)
all_dict.append(temp_dict)
newX = dv.fit_transform(all_dict)
# Separate the training and testing data sets
half_cut = int(len(df)/2.0)*-1
X_train = newX[:half_cut]
X_test = newX[half_cut:]
y_train = y[:half_cut]
y_test = y[half_cut:]
# Fitting X and y into model, using training data
lr = LogisticRegression()
lr.fit(X_train, y_train)
dv = DictVectorizer()
# Feature selection
plt.figure(1)
plt.clf()
X_indices = np.arange(X_train.shape[-1])
selector = SelectPercentile(f_classif, percentile=10)
selector.fit(X_train, y_train)
scores = -np.log10(selector.pvalues_)
scores /= scores.max()
plt.bar(X_indices - .45, scores, width=.2,
label=r'Univariate score ($-Log(p_{value})$)', color='g')
plt.show()
Warning:
E:\Program Files Extra\Python27\lib\site-packages\sklearn\feature_selection\univariate_selection.py:111: UserWarning: Features [[0 0 0 ..., 0 0 0]] are constant.
It looks like the way you split your data into training and testing sets is not working:
# Separate the training and testing data sets
X_train = newX[:half_cut]
X_test = newX[half_cut:]
If you already use sklearn, it is much more convenient to use the builtin splitting routine for this:
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.5, random_state=0)
I am Trying to call a another method within my class, for some reason I am getting the AttributeError: portfinder instance has no attribute 'generatePortNumber' See my code below:
when I tried to call the generatePortNumber I'm getting the same error. I have never come across this issue.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sqlite3 as lite
import sys
import random
class portfinder:
"""docstring for ClassName"""
def __init__(self):
self.portsToCheck = ['agentport','BatchProcessingAgentPort','databaseport','indexserviceport','monitorport','servicefacadeport','webdriverport']
self.dataBasePort = (u'60025',)
self.portInUse = False
self.x = 0
def generatePortNumber(self):
self.newPortNumber = random.randrange(8000, 9000)
print self.newPortNumber
return self.newPortNumber
def findUsedPortsinDB(self):
con = lite.connect('D:\play\Opes\db.sqlite3')
with con:
cur = con.cursor()
sqlStatement = "Select " + self.portsToCheck[2] +' From Jobs_jobs'
print sqlStatement
cur.execute(sqlStatement)
rows = cur.fetchall()
for row in rows:
print row
if row == self.dataBasePort:
self.portInUse = "true"
self.generatePortNumber()
if __name__ == "__main__":
m = portfinder()
m.findUsedPortsinDB()
Found what was wrong I had a extra indentation in my method