I have the following code:
def index(request):
events_list = Timesheet.objects.filter(owner = request.user.pk, week = datetime.datetime.now().isocalendar()[1]).order_by('-working_hour')
total_hours_week = Timesheet.objects.annotate(total_hours_per_week=Sum('working_hour')).filter(owner = request.user.pk, week = datetime.datetime.now().isocalendar()[1])
return render(request, "timesheet/index.html", {'events_list': events_list, 'total_hours_week': total_hours_week})
the total_hours_week retun the current error:
You cannot use Sum, Avg, StdDev, and Variance aggregations on date/time fields in sqlite3 since date/time is saved as text.
Do you know how to fix ?
thanks per advance
You can check if the current DB engine is sqlite3 or not, and calculate annotations programmatically in case that is needed:
from django.conf import settings
def index(request):
db_engine = settings.DATABASES['default']['ENGINE']
if db_engine = 'django.db.backends.sqlite3':
total_hours_week = [] # Calculate annotation programmatically
else:
total_hours_week = Timesheet.objects.annotate(total_hours_per_week=Sum('working_hour')).filter(owner = request.user.pk, week = datetime.datetime.now().isocalendar()[1])
events_list = Timesheet.objects.filter(owner = request.user.pk, week = datetime.datetime.now().isocalendar()[1]).order_by('-working_hour')
return render(request, "timesheet/index.html", {'events_list': events_list, 'total_hours_week': total_hours_week})
However, if you are using sqlite3 only for development, and you need to work with time aggregations a lot, I strongly recommend you to setup a PostgreSQL/MySql server for your development environment, as it saves you a lot of time and headache in long run.
Related
Timezones in Django...
I am not sure why this is so difficult, but I am stumped.
I have a form that is overwriting the UTC dateTime in the database with the localtime of the user. I can't seem to figure out what is causing this.
my settings.py timezone settings look like:
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'America/Toronto'
USE_I18N = True
USE_L10N = False
USE_TZ = True
I am in Winnipeg, my server is hosted in Toronto. My users can be anywhere.
I have a modelfield for each user that is t_zone = models.CharField(max_length=50, default = "America/Winnipeg",) which users can change themselves.
with respect to this model:
class Build(models.Model):
PSScustomer = models.ForeignKey(Customer, on_delete=models.CASCADE)
buildStart = models.DateTimeField(null=True, blank=True)
...
I create a new entry in the DB using view logic like:
...
now = timezone.now()
newBuild = Build(author=machine,
PSScustomer = userCustomer,
buildStart = now,
status = "building",
addedBy = (request.user.first_name + ' ' +request.user.last_name),
...
)
newBuild.save()
buildStart is saved to the database in UTC, and everything is working as expected. When I change a user's timezone in a view with timezone.activate(pytz.timezone(self.request.user.t_zone)) it will display the UTC time in their respective timezone.
All is good (I think) so far.
Here is where things go sideways:
When I want a user to change buildStart in a form, I can't seem to get the form to save the date to the DB in UTC. It will save to the DB in whatever timezone the user has selected as their own.
Using this form:
class EditBuild_building(forms.ModelForm):
buildStart = forms.DateTimeField(input_formats = ['%Y-%m-%dT%H:%M'],widget = forms.DateTimeInput(attrs={'type': 'datetime-local','class': 'form-control'},format='%Y-%m-%dT%H:%M'), label = "Build Start Time")
def __init__(self, *args, **kwargs):# for ensuring fields are not left empty
super(EditBuild_building, self).__init__(*args, **kwargs)
self.fields['buildDescrip'].required = True
class Meta:
model = Build
fields = ['buildDescrip', 'buildStart','buildLength'...]
labels = {
'buildDescrip': ('Build Description'),
'buildStart': ('Build Start Time'),
...
}
widgets = {'buildDescrip': forms.TextInput(attrs={'class': 'required'}),
and this view:
class BuildUpdateView_Building(LoginRequiredMixin,UpdateView):
model = Build
form_class = EditBuild_building
template_name = 'build_edit_building.html'
login_url = 'login'
def get(self, request, *args, **kwargs):
proceed = True
try:
instance = Build.objects.get(id = (self.kwargs['pk']))
except:
return HttpResponse("<h2 style = 'margin:2em;'>This build is no longer available it has been deleted, please please return to dashboard</h2>")
if instance.buildActive == False:
proceed = False
if instance.deleted == True:
proceed = False
#all appears to be well, process request
if proceed == True:
form = self.form_class(instance=instance)
timezone.activate(pytz.timezone(self.request.user.t_zone))
customer = self.request.user.PSScustomer
choices = [(item.id, (str(item.first_name) + ' ' + str(item.last_name))) for item in CustomUser.objects.filter(isDevice=False, PSScustomer = customer)]
choices.insert(0, ('', 'Unconfirmed'))
form.fields['buildStrategyBy'].choices = choices
form.fields['buildProgrammedBy'].choices = choices
form.fields['operator'].choices = choices
form.fields['powder'].queryset = Powder.objects.filter(PSScustomer = customer)
context = {}
context['buildID'] = self.kwargs['pk']
context['build'] = Build.objects.get(id = (self.kwargs['pk']))
return render(request, self.template_name, {'form': form, 'context': context})
else:
return HttpResponse("<h2 style = 'margin:2em;'>This build is no longer editable here, or has been deleted, please return to dashboard</h2>")
def form_valid(self, form):
timezone.activate(pytz.timezone(self.request.user.t_zone))
proceed = True
try:
instance = Build.objects.get(id = (self.kwargs['pk']))
except:
return HttpResponse("<h2 style = 'margin:2em;'>This build is no longer available it has been deleted, please please return to dashboard</h2>")
if instance.buildActive == False:
proceed = False
if instance.deleted == True:
proceed = False
#all appears to be well, process request
if proceed == True:
form.instance.editedBy = (self.request.user.first_name)+ " " +(self.request.user.last_name)
form.instance.editedDate = timezone.now()
print('edited date ' + str(form.instance.editedDate))
form.instance.reviewed = True
next = self.request.POST['next'] #grabs prev url from form template
form.save()
build = Build.objects.get(id = self.kwargs['pk'])
if build.buildLength >0:
anticipated_end = build.buildStart + (timedelta(hours = float(build.buildLength)))
print(anticipated_end)
else:
anticipated_end = None
build.anticipatedEnd = anticipated_end
build.save()
build_thres_updater(self.kwargs['pk'])#this is function above, it updates threshold alarm counts on the build
return HttpResponseRedirect(next) #returns to this page after valid form submission
else:
return HttpResponse("<h2 style = 'margin:2em;'>This build is no longer available it has been deleted, please please return to dashboard</h2>")
When I open this form, the date and time of buildStart are displayed in my Winnipeg timezone, so Django converted from UTC to my timezone, perfect, but when I submit this form, the date in the DB has been altered from UTC to Winnipeg Time. Why is this?
I have tried to convert the submitted time to UTC in the form_valid function, but this does not seem like the right approach. What am I missing here?
I simply want to store all times as UTC, but display them in the user's timezone in forms/pages.
EDIT
When I remove timezone.activate(pytz.timezone(self.request.user.t_zone)) from both get and form_valid, UTC is preserved in the DB which is great. But the time displayed on the form is now in the default TIME_ZONE in settings.py. I just need this to be in the user's timezone....
EDIT 2
I also tried to add:
{% load tz %}
{% timezone "America/Winnipeg" %}
{{form}}
{% endtimezone %}
Which displayed the time on the form correctly, but then when the form submits, it will again remove 1 hour from the UTC time in the DB.
If I change template to:
{% load tz %}
{% timezone "Europe/Paris" %}
{{form}}
{% endtimezone %}
The time will be displayed in local Paris time. When I submit the form, it will write this Paris time to the DB in UTC+2. So, in summary:
Time record was created was 11:40 Winnipeg time, which writes
16:40 UTC to database, perfect
I access the form template, and time is displayed as local Paris time, 6:40pm, which is also what I would expect.
I submit form without changing any fields.
Record has been updated with the time as 22:40, which is UTC + 6 hours.
What is happening here!?
Put simply: your activate() call in form_valid() comes too late to affect the form field, so the incoming datetime gets interpreted in the default timezone—which in your case is America/Toronto—before being converted to UTC and saved to the database. Hence the apparent time shift.
The documentation doesn't really specify when you need to call activate(). Presumably, though, it has to come before Django converts the string value in the request to the aware Python datetime in the form dictionary (or vice versa when sending a datetime). By the time form_valid() is called, the dictionary of field values is already populated with the Python datetime object.
The most common place to put activate() is in middleware (as in this example from the documentation), since that ensures that it comes before any view processing. Alternatively, if using generic class-based views like you are, you could put it in dispatch().
I'm currently using a Docker & Django setup. I have to fill a database with data from API requests. I was hoping to do this everytime you went on a certain page (pretty easy: just have your views.py call the function that fills the database and voila).
But the problem is, the function takes a long time, several minutes from within django (and about half the time with Spyder).
So I usually just get a TimeOut and the page never loads (I admit I have a lot of API requests being made).
I've read some stuff on using Celery but am not quite sure how it's supposed to work.
Anyone know how I could get around this to be able to load the database?
Edit: some code
Views.py
def index(request):
fill_db()
context = {}
context['segment'] = 'index'
html_template = loader.get_template( 'index.html' )
return HttpResponse(html_template.render(context, request))
fill_db function
def fill_db():
fill_agencies()
fill_companies()
fill_contracts()
fill_orders()
fill_projects()
fill_resources()
Example of a fill function:
r = pip._vendor.requests.get(BASE_URL+EXTENSION,auth=(USER,PASS))
data0 = json.loads(r.text)
conn = sqlite3.connect('/app/database.sqlite3')
c = conn.cursor()
for client in data0['data']:
BoondID = client['id']
name = client['attributes']['name']
expertiseArea = client['attributes']['expertiseArea']
town = client['attributes']['town']
country = client['attributes']['country']
mainManager = client['relationships']['mainManager']['data']['id']
values = (BoondID, name, expertiseArea, town, country, mainManager)
c.execute("INSERT OR REPLACE INTO COMPANIES (BoondID,name,expertiseArea,town,country,mainManager) VALUES (?,?,?,?,?,?);", values)
conn.commit()
conn.close()
Solved.
I used python's threading library.
I defined
agencies_thread = threading.Thread(target=fill_agencies, name="Database Updater")
and called agencies_thread.start() inside my views function.
This works fine.
I am building a Django webapp including a view, which can upload data into the database through a csv-import. Each import contains around 2,000 rows and 9 columns with DecimalFields and CharFields. So far I've been using Django's SQLite Database and each upload took me 1 min max. I switched to PostgreSQL (hosted via ElephantSQL) and now the upload takes at least 10 minutes. I've read in some posts that SQLite is faster than PostgreSQL but I was not expecting anything of this magnitude. Is there a way to speed up the upload process in PostgreSQL? I thought one reason for the low speed might be that I am using ElephantSQL's free Tiny Turtle Plan, but if i understand correctly the non-free plans differ only in terms of the max size of the database but not its speed? See also here https://www.elephantsql.com/plans.html
Might it be a solution to have PostgreSQL installed locally instead of using a cloud provider? Is there anything else I can optimize to speed up the process?
my model:
class Testdata3(models.Model):
key = models.CharField(max_length=100, primary_key=True)
mnemonic = models.CharField(max_length=50)
assetclass = models.CharField(max_length=50)
value = models.DecimalField(max_digits=255, decimal_places=25)
performance = models.DecimalField(max_digits=255, decimal_places=25)
performance_exccy = models.DecimalField(max_digits=255, decimal_places=25)
performance_abs = models.DecimalField(max_digits=255, decimal_places=25)
performance_abs_exccy = models.DecimalField(max_digits=255, decimal_places=25)
date = models.DateField()
def __str__(self):
return self.key
my view:
def file_upload(request):
template = "upload.html"
prompt = {
'order': 'Order of the CSV should be "placeholder_1", "placeholder_2", "placeholder_3" '
}
if request.method == "GET":
return render(request, template, prompt)
csv_file = request.FILES['file']
if not csv_file.name.endswith('.csv'):
messages.error(request, 'This is not a csv file')
data_set = csv_file.read().decode('UTF-8')
io_string = io.StringIO(data_set)
#Ignores header row by jumping to next row
next(io_string)
for column in csv.reader(io_string, delimiter=';', quotechar="|"):
# Check if csv-row is empty, if true jump to next iteration/row
if all(elem == "" for elem in column):
next
else:
_, created = Testdata3.objects.update_or_create(
key = column[0],
defaults = {
'key' : column[0],
# Get everything after the date part in the primary key
'mnemonic': re.findall(r'AMCS#[0-9]*(.*)', column[0])[0],
# Create datetime object from a string
'date' : datetime.datetime.strptime(column[6], '%d/%m/%Y'),
'assetclass' : column[10],
'value' : column[16],
'performance' : column[19],
'performance_abs' : column[20],
'performance_abs_exccy' : column[30],
'performance_exccy' : column[31],
}
)
context = {}
return render(request, template, context)
I don't think so. I guess there is some problem with your service provider, or the CSV file you are importing is very large. I use AWS RDS with Postgres and that is fast enough. It is nothing related to SQLite vs Postgres. Also, it can be because of your disk's IO speed which can be very high for SSD and high-end machines.
I'm building a basic blog from the Web Development course by Steve Hoffman on Udacity. This is my code -
import os
import webapp2
import jinja2
from google.appengine.ext import db
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader = jinja2.FileSystemLoader(template_dir), autoescape = True)
def datetimeformat(value, format='%H:%M / %d-%m-%Y'):
return value.strftime(format)
jinja_env.filters['datetimeformat'] = datetimeformat
def render_str(template, **params):
t = jinja_env.get_template(template)
return t.render(params)
class Entries(db.Model):
title = db.StringProperty(required = True)
body = db.TextProperty(required = True)
created = db.DateTimeProperty(auto_now_add = True)
class MainPage(webapp2.RequestHandler):
def get(self):
entries = db.GqlQuery('select * from Entries order by created desc limit 10')
self.response.write(render_str('mainpage.html', entries=entries))
class NewPost(webapp2.RequestHandler):
def get(self):
self.response.write(render_str('newpost.html', error=""))
def post(self):
title = self.request.get('title')
body = self.request.get('body')
if title and body:
e = Entries(title=title, body=body)
length = db.GqlQuery('select * from Entries order by created desc').count()
e.put()
self.redirect('/newpost/' + str(length+1))
else:
self.response.write(render_str('newpost.html', error="Please type in a title and some content"))
class Permalink(webapp2.RequestHandler):
def get(self, id):
e = db.GqlQuery('select * from Entries order by created desc').get()
self.response.write(render_str('permalink.html', id=id, entry = e))
app = webapp2.WSGIApplication([('/', MainPage),
('/newpost', NewPost),
('/newpost/(\d+)', Permalink)
], debug=True)
In the class Permalink, I'm using the get() method on the query than returns all records in the descending order of creation. So, it should return the most recently added record. But when I try to add a new record, permalink.html (it's just a page with shows the title, the body and the date of creation of the new entry) shows the SECOND most recently added. For example, I already had three records, so when I added a fourth record, instead of showing the details of the fourth record, permalink.html showed me the details of the third record. Am I doing something wrong?
I don't think my question is a duplicate of this - Read delay in App Engine Datastore after put(). That question is about read delay of put(), while I'm using get(). The accepted answer also states that get() doesn't cause any delay.
This is because of eventual consistency used by default for GQL queries.
You need to read:
https://cloud.google.com/appengine/docs/python/datastore/data-consistency
https://cloud.google.com/appengine/docs/python/datastore/structuring_for_strong_consistency
https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/
search & read on SO and other source about strong & eventual consistency in Google Cloud Datastore.
You can specify read_policy=STRONG_CONSISTENCY for your query but it has associated costs that you should be aware of and take into account.
My application is very simple, it has a WeekArchiveView class based view:
class MenuWeekArchiveView(WeekArchiveView):
queryset = Menu.objects.all()
And its corresponding URL:
url(r'^(?P<year>[0-9]{4})/week/(?P<week>[0-9]+)/$', menu.views.MenuWeekArchiveView.as_view(), name="menu_week"),
I would like to have the home page of my application return the current week.
With old function based views, this was easy. I'd just have the home page return that function with the current week number as the arguments.
today = datetime.date.today()
current_week_number = today.isocalendar()[1]
current_year = today.year
return week_view(request, year=current_year, week=current_week_number)
A redirect wouldn't be acceptable because when someone bookmarks the page, they'll be bookmarking that week.
View.as_view() returns a proper view function you can use:
today = datetime.date.today()
current_week_number = today.isocalendar()[1]
current_year = today.year
return MenuWeekArchiveView.as_view()(request, year=current_year, week=current_week_number)