Problem scale：I have 100000 variables and 3 constraints.
Ten hours have passed，the above problem was not solved.
How to optimize the problem.
pyomo solve pp.py --solver=glpk
model = ConcreteModel()
model.I = Set(initialize = [i for i in range(N)])
model.x = Var(model.I, within = NonNegativeIntegers, bounds = (0,1))
model.pctr = Param(model.I, within = NonNegativeReals, initialize=data.pctr.to_dict()) # pctri
model.pcvr = Param(model.I, within = NonNegativeReals, initialize=data.pcvr.to_dict()) # pcvri
model.mkt_ecpm = Param(model.I, within=NonNegativeReals, initialize=data.mkt_ecpm.to_dict(), default=0.0) # mkti
model.v = Param(model.I, within=NonNegativeReals, initialize=v_init_click, default=0.0)
model.c = Param(model.I, within=NonNegativeReals, initialize=c_init, default=0.0)
def constrs_budget(model,i):
return sum(model.x[i]*model.c[i] for i in model.I) <= budget
model.constrs_budget = Constraint(model.I, rule=constrs_budget)
def constrs_cpc(model, i):
return sum(model.x[i]*(model.c[i]-cpc*model.v[i]) for i in model.I) <= 0.0
model.constrs_cpc = Constraint(model.I, rule=constrs_cpc)
model.constrs_x = Constraint(model.I, rule=constrs_x)
model.obj = Objective(rule=obj_cpi, sense=maximize)
Related
I am trying to build a simple form which calculates which machine would run a film width the quickest, the parameters and capabilities of each machine are held in a django model.
The width of the film and how much of it will be entered in the form and the quantity needed. The function should work out which machine(s) can run it, what the max speed is and the average speed over the machines that are capable.
I want to return the values of the calculation and maybe run a for loop and display the values for each machine in a results.html template in a table. I also want to display average times across machines capable of running the widths of film.
I had some success with lists but would like to use a class that I can use in the template and do away with the lists.
Any help with this would be much appreciated as I am getting pretty lost in it!
I have only started on the 'Layflat Tubing' function in the hope that I can get it right and just copy down to the other functions.
from django.views.generic.base import TemplateView
from django.shortcuts import render
import math, datetime
from settings.models import Extruder
class Result:
def __init__(self, ext_no, width, speed=0, ):
self.ext_no = ext_no
self.width = width
self.speed = speed
def __str__(self):
return self.ext_no
extruders = Extruder.objects.all()
class FilmSpeedView(TemplateView):
template_name = 'calculations/film-speed.html'
class BagWeightView(TemplateView):
template_name = 'calculations/bag-weight.html'
class CalculatorsView(TemplateView):
template_name = 'calculations/calculators.html'
def result(request):
film_type=''
film_width=''
measure=''
speeds = [0]
quantity = 0
max_speed = 0
ave_speed = 0
ave_time = 0
max_time = 0
a=[]
b=[]
c=[]
d=[]
e=[]
if request.method=="GET":
film_type = str(request.GET["film_type"])
film_width = int(request.GET["film_width"])
edge_trim = int(request.GET["edge_trim"])
quantity =int(request.GET["quantity"])
measure = str(request.GET["measure"])
if measure == "metric":
film_width = int(film_width)
else:
film_width = film_width * 25.4
if edge_trim is None:
edge_trim = 0
else:
edge_trim = int(edge_trim)
if str(film_type) == 'Layflat Tubing':
film_type = "LFT"
for extruder in extruders:
bur = film_width / extruder.die_size
if film_width < extruder.min_width:
b.append(extruder.name + ' : Film too narrow')
extruder = Result(ext_no = extruder.ext_no, width = 'too narrow')
elif film_width > extruder.max_width:
b.append(extruder.name + ' : Film too wide')
extruder = Result(ext_no = extruder.ext_no, width = 'too wide')
else:
percentage = film_width / extruder.max_width
speed = extruder.max_kgs_hr * percentage
extruder = Result(ext_no = extruder.ext_no, speed = round(extruder.max_kgs_hr * percentage, 2), width = 'ok')
speeds.append(speed)
max_speed = max(speeds)
ave_speed = sum(speeds) / len(speeds)
ave_time = float(quantity) / ave_speed * 60.0
max_time = float(quantity) / max_speed * 60.0
else:
film_type = "Invalid Film Type"
m = a
n = b
o = c
g = str(round(ave_speed, 2)) + 'kg\'s/h'
h = str(datetime.timedelta(minutes=ave_time))
i = str(datetime.timedelta(minutes=30))
j = str(round(max_speed, 2)) + 'kg\'s/h'
k = str(datetime.timedelta(minutes=max_time))
return render(request, 'calculations/result.html', {'a':a, 'b':b, 'c':c, 'd':d, 'e':e, 'g':g, 'h':h, 'i':i, 'j':j, 'k':k, 'm':m, 'n':n, 'o':o, 'bur':bur,})
I've created a form for being stored ratings and feedback in the database. Ratings and Feedback are being stored in the database perfectly. But the problem is, I can't find out how many different types of rating stars are present in the database. How can I find out how many 1 star/2star/---5 stars are present in the object model in percentage? What should I do?
models.py:
class Frontend_Rating(models.Model):
USer = models.ForeignKey(User,default=None,on_delete=models.CASCADE, related_name="frontend_rating")
Rating = models.IntegerField(null=True)
Feedback = models.TextField(max_length=250, null=True)
def __str__(self):
return str(self.pk)+ str(".") + str(self.USer) + str("(") + str(self.Rating) + str("stars") +str(")")
Views.py:
def index(request):
#rating____________________
frontend_all_ratings = Frontend_Rating.objects.all()
number_of_frontend_rating = frontend_all_ratings.count()
average_rating = 0
frontend_one_star = 0
frontend_two_star = 0
frontend_three_star = 0
frontend_four_star = 0
frontend_five_star = 0
percentage_frontend_ONE_star = 0
percentage_frontend_FIVE_star = 0
for frontend_rating_item in frontend_all_ratings:
frontend_rating = frontend_rating_item.Rating
if frontend_rating:
total_ratings = 0
total_ratings += frontend_rating
average_rating = round(total_ratings/frontend_all_ratings.count(),1)
context = {
"frontend_five_star":frontend_five_star,
"frontend_one_star":frontend_one_star,
"total_ratings":total_ratings,
"average_rating":average_rating,
}
return render(request,'0_index.html',context)
You can obtain the data with a single query with:
from django.db.models import Avg, BooleanField, ExpressionWrapper, Q
data = Frontend_Rating.objects.aggregate(
frontend_one_star=Avg(ExpressionWrapper(Q(Rating=1), output_field=BooleanField())),
frontend_two_star=Avg(ExpressionWrapper(Q(Rating=2), output_field=BooleanField())),
frontend_three_star=Avg(ExpressionWrapper(Q(Rating=3), output_field=BooleanField())),
frontend_four_star=Avg(ExpressionWrapper(Q(Rating=4), output_field=BooleanField())),
frontend_five_star=Avg(ExpressionWrapper(Q(Rating=5), output_field=BooleanField())),
)
frontend_one_star = data['frontend_one_star']
frontend_two_star = data['frontend_two_star']
frontend_three_star = data['frontend_three_star']
frontend_four_star = data['frontend_four_star']
frontend_five_star = data['frontend_five_star']
or for databases that use integer division, like postgresql:
from django.db.models import Count, FloatField, Q
data = Frontend_Rating.objects.aggregate(
frontend_one_star=Count('pk', filter=Q(Rating=1), output_field=FloatField()) / Count('pk', output_field=FloatField()),
frontend_two_star=Count('pk', filter=Q(Rating=2), output_field=FloatField()) / Count('pk', output_field=FloatField()),
frontend_three_star=Count('pk', filter=Q(Rating=3), output_field=FloatField()) / Count('pk', output_field=FloatField()),
frontend_four_star=Count('pk', filter=Q(Rating=4), output_field=FloatField()) / Count('pk', output_field=FloatField()),
frontend_five_star=Count('pk', filter=Q(Rating=5), output_field=FloatField()) / Count('pk', output_field=FloatField()),
)
frontend_one_star = data['frontend_one_star']
frontend_two_star = data['frontend_two_star']
frontend_three_star = data['frontend_three_star']
frontend_four_star = data['frontend_four_star']
frontend_five_star = data['frontend_five_star']
Note: Models in Django are written in PascalCase, not snake_case,
so you might want to rename the model from Frontend_Rating to FrontendRating.
Note: normally the name of the fields in a Django model are written in snake_case, not PascalCase, so it should be: rating instead of Rating.
number_of_frontend_rating = Frontend_Rating.objects.count()
# Divide value by overall count to get ratio and multiply ratio by 100 to get percentage
frontend_one_star = (Frontend_Rating.objects.filter(Rating=1).count() / overall_count)*100
frontend_two_star = (Frontend_Rating.objects.filter(Rating=2).count() / overall_count)*100
frontend_three_star = (Frontend_Rating.objects.filter(Rating=3).count() / overall_count)*100
frontend_four_star = (Frontend_Rating.objects.filter(Rating=4).count() / overall_count)*100
frontend_five_star = (Frontend_Rating.objects.filter(Rating=5).count() / overall_count)*100
Please correct me if I misunderstood your question
I want to use boost.python to use multi-index columns dataframe in c++.
※multi-index columns dataframe is like
I changed the type of multi-index columns dataframe into csv.
My csv file looks like this on spreadsheet
The reason why I want to use this data is for backtest. This is my backtest code in python that I want to translate to c++.
import pandas as pd
import numpy as np
from utils import load_data, load_list_csv, to_int
class No_Strategy():
def __init__(self, codes, unit, cash, position):
self.codes = codes
self.unit = unit
self.cash = cash
self.buy_signal = [0]*len(codes)
self.sell_signal = [0]*len(codes)
self.valid = 0
self.position = position
self.pass_st = 0 # 전략에 들어가지도 못한 경우
def set_data(self, prev_fs_row, fs_row, indi_row):
self.prev_fs = prev_fs_row
self.fs = fs_row # multi dimensional df
self.indi = indi_row
def _strat(self, prev_fs, curr_fs, curr_indi):
curr_rev = prev_rev = curr_ni = prev_ni = ni_growth = curr_asset = noncurr_asset = curr_asset_rat = 0
try:
prev_rev = int(prev_fs['매출액'].replace(",",""))
curr_rev = int(curr_fs['매출액'].replace(",",""))
except:
self.pass_st += 1
return 0, 0
rev_growth=(curr_rev-prev_rev)/prev_rev
try:
prev_ni = int(prev_fs['당기순이익'].replace(",",""))
curr_ni = int(curr_fs['당기순이익'].replace(",",""))
except:
self.pass_st += 1
return 0, 0
ni_growth=(curr_ni-prev_ni)/prev_ni
try:
curr_asset = int(curr_fs['유동자산'].replace(",",""))
noncurr_asset = int(curr_fs['비유동자산'].replace(",",""))
except:
self.pass_st += 1
return 0, 0
curr_asset_rat = curr_asset / noncurr_asset
#### this is the buy strategy! You can change the below ####
if (curr_indi.golden_cross) or (curr_indi.rsi_k < 0.65) :
return 1, 0
#### ************************************************** ####
#### this is the sell strategy! You can change the below ####
if (curr_indi.dead_cross):
return 0, 1
#### ************************************************** ####
return 0, 0
def run(self):
for i, code in enumerate(self.codes):
self.valid = 0
prev_fs = self.prev_fs[code]
curr_fs = self.fs[code]
curr_indi = self.indi[code]
prev_fs_cell = None
curr_fs_cell = None
try:
prev_fs_cell = prev_fs.iloc[0].replace(",","")
try:
curr_fs_cell = curr_fs.iloc[0].replace(",","")
except:
self.pass_st += 1
pass
except:
self.pass_st += 1
pass
if (curr_fs_cell != None) & (prev_fs_cell != None):
self.valid = 1
buy, sell = self._strat(prev_fs, curr_fs, curr_indi)
if self.valid == 0:
self.pass_st += 1
continue
else: # buy or sell signal get
price = curr_indi['close']
if buy:
if self.cash >= self.unit * price:
self.buy_signal[i] = self.unit
self.position[i] += self.unit
self.cash -= price * self.unit
elif sell:
if self.position[i] > 0 :
sell_num = self.position[i] - int(self.position[i]/2)
self.sell_signal[i] = sell_num
self.position[i] = int(self.position[i]/2) # 1-> 1 sell, 4 -> 2 sell ....
self.cash += price * sell_num
##title
class Broker():
def __init__(self, codes):
self.cash = 200000000 #2억
self.cash_df = None #pd.DataFrame(columns=['cash'])
self.position = [0]*len(codes)
self.position_df = None #pd.DataFrame(columns=codes) # for accumulated profit calculation
self.buy_signal = None #pd.DataFrame(columns=codes) # codes = KOSPI_stock_names
self.sell_signal = None #pd.DataFrame(columns=codes)
self.codes = codes # 012934, 3281, ...
self.unit = 1 # 주식 매매 단위
self.pass_st = 0
def set_strat(self, strategy):
self.strategy = strategy # class
def set_time(self, time_index): # time_index type: pd.Index / time range for indi df
self.buy_signal = pd.DataFrame(columns = self.codes, index = time_index) #set_index(time_index)
self.sell_signal = pd.DataFrame(columns = self.codes, index = time_index) #.set_index(time_index)
self.position_df = pd.DataFrame(columns = self.codes, index = time_index)
self.cash_df = pd.DataFrame(columns = ['cash'], index = time_index)#.set_index(time_index)
self.time_index = time_index
def set_data(self, fs, indi, price):
self.fs = fs # multi dimensional df / start: 0th - nth
self.indi = indi # multi dimensional df / start : 1th - nth
self.price = price # 2 dimensional (date X codes : close price)
def update_data(self, strategy, date):
self.cash = strategy.cash
self.cash_df.loc[date] = strategy.cash
self.position = strategy.position
self.position_df.loc[date] = strategy.position #list
self.buy_signal.loc[date] = strategy.buy_signal #list
self.sell_signal.loc[date] = strategy.sell_signal #list
self.pass_st += strategy.pass_st
def run(self):
for date in self.time_index: #아마 수정해야 할 확률 높음
if date.year == 2021:
break
else:
prev_fs_row = self.fs.loc[date.year-1] # ex: 2014
fs_row = self.fs.loc[date.year] # 2015
indi_row = self.indi.loc[date] # 2015
strategy = self.strategy(self.codes, self.unit, self.cash, self.position)
strategy.set_data(prev_fs_row, fs_row, indi_row)
strategy.run()
self.update_data(strategy, date)
def performance(self):
# !!!! 2020년까지의 결과만 성능 평가 ####
cash_df = self.cash_df[self.cash_df.index < '2021']
position_df = self.position_df[self.position_df.index < '2021']
price = self.price[self.price.index < '2021']
buy_signal = self.buy_signal[self.buy_signal.index < '2021']
sell_signal = self.sell_signal[self.sell_signal.index < '2021']
last_price = price.iloc[-1]
total_remain_num = self.position # last(2020) position data
total_buy = (price * buy_signal).sum(axis=1).sum()
total_sell = (price * sell_signal).sum(axis=1).sum()
total_remain = (last_price * total_remain_num).sum()
print(f'remain 개수: {total_remain_num}, total_remain: {total_remain} total_buy: {total_buy}, total_sell={total_sell}')
profit = total_sell + total_remain - total_buy
try:
return_mean = profit / total_buy
except:
print("no buy")
return
accum_df = (cash_df['cash'] + ((price.fillna(0) * position_df).sum(axis=1))).to_frame() # row sum
daily_return_df = (accum_df - accum_df.shift(1))/accum_df.shift(1)-1
SSE = ((daily_return_df - return_mean)**2).sum().item()
std = np.sqrt(SSE/(accum_df.shape[0]-1)) # route(sigma(x-x_bar)^2 / (n-1))
sharp = return_mean / std
self.return_mean = return_mean
self.sharp = sharp
print(f'return_mean: {return_mean}, sharp: {sharp}')
code_path = GDRIVE_DATA_PATH + 'codes.csv'
fs_path = GDRIVE_DATA_PATH + 'fs_total.csv'
indi_path = GDRIVE_DATA_PATH + 'indi_total.csv'
price_path = GDRIVE_DATA_PATH + 'prices.csv'
fs_total = load_data("fs_total.csv")
indi_total = load_data("indi_total.csv") # stock price and indicator(Golden cross, RSI, etc.)
prices = load_data("prices.csv") # stock close price data rows:date, cols: stock code.
time_index = indi_total.index # time index of indi_total multi-index columns
broker = Broker(codes)
broker.set_strat(No_Strategy)
broker.set_time(time_index)
broker.set_data(fs_total, indi_total, prices)
broker.run()
broker.performance()
I want to translate it not changing much in flow of the code.
But I cannot find how to get multi-index columns dataframe in c++, and transfer its row data to No_Strategy to decide whether invest into the stock.
※ I uploaded similar question before and get thankful answer, but it is too complicated for me so I question one more time with detail information.
look at https://github.com/hosseinmoein/DataFrame. It has about 95% of Pandas functionality in a much faster framework
I have been using in my machine a network, that is nothing really special. I wanted to do it faster so I started using google cloud. But I notice something weird that my machine with a GTX 1050 ti was faster than a V100 GPU. This didn't add up so I checked the usage and it seems that even though I put some stress by creating a big network and passing a lot of data to it the gpu by using a simple .cuda() in both the model and the data: there wasn't ussage shown in nvidia-smi command as shown in the image
you can check my code here:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("The device is:",device,torch.cuda.get_device_name(0),"and how many are they",torch.cuda.device_count())
# # We load the training data
Samples , Ocupancy, num_samples, Samples_per_slice = common.load_samples(args.samples_filename)
Samples = Samples * args.scaling_todo
print(Samples_per_slice)
# Divide into Slices
Organize_Positions,Orginezed_Ocupancy, batch_size = common.organize_sample_data(Samples,Ocupancy,num_samples,Samples_per_slice,args.num_batches)
phi = common.MLP(3, 1).cuda()
x_test = torch.from_numpy(Organize_Positions.astype(np.float32)).cuda()
y_test = torch.from_numpy(Orginezed_Ocupancy.astype(np.float32)).cuda()
all_data = common.CustomDataset(x_test, y_test)
#Dive into Slices the data
Slice_data = DataLoader(dataset=all_data, batch_size = batch_size, shuffle=False) # only take batch_size = n/b TODO Don't shuffle
#Chunky_data = DataLoader(dataset=Slice_data, batch_size = chunch_size, shuffle=False)
criterion = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(phi.parameters(), lr = 0.0001)
epoch = args.num_epochs
fit_start_time = time.time()
phi.train()
for epoch in range(epoch):
curr_epoch_loss = 0
batch = 0
for x_batch, y_batch in Slice_data:
optimizer.zero_grad()
x_train = x_batch
#print(x_train,batch_size)
y_train = y_batch
y_pred = phi(x_train)
#print(y_pred,x_train)
loss = criterion(y_pred.squeeze(), y_train.squeeze())
curr_epoch_loss += loss
print('Batch {}: train loss: {}'.format(batch, loss.item())) # Backward pass
loss.backward()
optimizer.step() # Optimizes only phi parameters
batch+=1
print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
fit_end_time = time.time()
print("Total time = %f" % (fit_end_time - fit_start_time))
# Save Model
torch.save({'state_dict': phi.state_dict()}, args.model_filename)
and the model here:
class MLP(nn.Module):
def __init__(self, in_dim: int, out_dim: int):
super().__init__()
self.in_dim = in_dim
self.out_dim = out_dim
self.fc1 = nn.Linear(in_dim, 128)
self.fc1_bn = nn.BatchNorm1d(128)
self.fc2 = nn.Linear(128, 256)
self.fc2_bn = nn.BatchNorm1d(256)
self.fc3 = nn.Linear(256, 512)
self.fc3_bn = nn.BatchNorm1d(512)
self.fc4 = nn.Linear(512, 512)
self.fc4_bn = nn.BatchNorm1d(512)
self.fc5 = nn.Linear(512, out_dim,bias=False)
self.relu = nn.LeakyReLU()
def forward(self, x):
x = self.relu(self.fc1_bn(self.fc1(x)))
x = self.relu(self.fc2_bn(self.fc2(x)))# leaky
x = self.relu(self.fc3_bn(self.fc3(x)))
x = self.relu(self.fc4_bn(self.fc4(x)))
x = self.fc5(x)
return x
class CustomDataset(Dataset):
def __init__(self, x_tensor, y_tensor):
self.x = x_tensor
self.y = y_tensor
def __getitem__(self, index):
return (self.x[index], self.y[index])
def __len__(self):
return len(self.x)
I am trying migrating the ampl car problem that comes in the Ipopt source code tarball as example. I am having got problems with the end condition (reach a place with zero speed at final iteration) and with the cost function (minimize final time).
Can someone help me revise the following model?
# min tf
# dx/dt = 0
# dv/dt = a - R*v^2
# x(0) = 0; x(tf) = 100
# v(0) = 0; v(tf) = 0
# -3 <= a <= 1 (a is the control variable)
#!Python3.5
from pyomo.environ import *
from pyomo.dae import *
N = 20;
T = 10;
L = 100;
m = ConcreteModel()
# Parameters
m.R = Param(initialize=0.001)
# Variables
def x_init(m, i):
return i*L/N
m.t = ContinuousSet(bounds=(0,1000))
m.x = Var(m.t, bounds=(0,None), initialize=x_init)
m.v = Var(m.t, bounds=(0,None), initialize=L/T)
m.a = Var(m.t, bounds=(-3.0,1.0), initialize=0)
# Derivatives
m.dxdt = DerivativeVar(m.x, wrt=m.t)
m.dvdt = DerivativeVar(m.v, wrt=m.t)
# Objetives
m.obj = Objective(expr=m.t[N])
# DAE
def _ode1(m, i):
if i==0:
return Constraint.Skip
return m.dxdt[i] == m.v[i]
m.ode1 = Constraint(m.t, rule=_ode1)
def _ode2(m, i):
if i==0:
return Constraint.Skip
return m.dvdt[i] == m.a[i] - m.R*m.v[i]**2
m.ode2 = Constraint(m.t, rule=_ode2)
# Constraints
def _init(m):
yield m.x[0] == 0
yield m.v[0] == 0
yield ConstraintList.End
m.init = ConstraintList(rule=_init)
'''
def _end(m, i):
if i==N:
return m.x[i] == L amd m.v[i] == 0
return Constraint.Skip
m.end = ConstraintList(rule=_end)
'''
# Discretize
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(m, nfe=N, wrt=m.t, scheme='BACKWARD')
# Solve
solver = SolverFactory('ipopt', executable='C:\\EXTERNOS\\COIN-OR\\win32-msvc12\\bin\\ipopt')
results = solver.solve(m, tee=True)
Currently, a ContinuousSet in Pyomo has to be bounded. This means that in order to solve a minimum time optimal control problem using this tool, the problem must be reformulated to remove the time scaling from the ContinuousSet. In addition, you have to introduce an extra variable to represent the final time. I've added an example to the Pyomo github repository showing how this can be done for your problem.