I have a model defined similar to below
class MyModel(models.Model):
num_attempts = models.IntegerField()
num_generated = models.IntegerField()
num_deleted = models.IntegerField()
Assuming my data looked something like this:
|id|num_attempts|num_generated|num_deleted
1 1 2 0
2 2 0 1
3 3 2 1
4 3 1 2
I want to get a count of the instances at each possible value for each possible field.
For example, a return sample could look like this.
{
'num_attempts_at_1': 1,
'num_attempts_at_2': 1,
'num_attempts_at_3': 2,
'num_generated_at_0': 1,
'num_generated_at_1': 1,
'num_generated_at_2': 2,
'num_deleted_at_0': 1,
'num_deleted_at_1': 2,
'num_deleted_at_2': 1
}
This above example assumes a lot, like naming of the variables after and that it would be serialized. None of that matters but rather just how do I get it broken down like that from the database. It would be best to have this done in one query if possible.
We are using Postgres as the database.
Here is sorta close, but not quite.
qs.values_list('num_attempts','num_generated','num_deleted').annotate(Count('id'))
Gives this (not the same data as the example above)
[{'num_attempts': 4, 'id__count': 3, 'num_deleted': 3, 'num_generated': 6}, {'num_attempts': 3, 'id__count': 12, 'num_deleted': 2, 'num_generated': 2}, {'num_attempts': 2, 'id__count': 5, 'num_deleted': 0, 'num_generated': 6}]
Now with some custom python I was able to do this, but really want a database solution if possible.
def get(self, request, *args, **kwargs):
qs = self.get_queryset()
return_data = {}
for obj in qs:
count = obj.pop('id__count')
for k, v in obj.items():
key = "{}_at_{}".format(k, v)
value = return_data.get(key, 0) + count
return_data[key] = value
return Response(return_data)
I need to remove the unique elements of the list, the first thought is:
def cut_uniq(data):
for x in data:
if data.count(x) == 1:
data.remove(x)
print(data)
cut_uniq([1, 2, 3, 4, 5,])
return
[2, 4]
please, tell me why?
Look at each iteration:
i x data
0 1 [1,2,3,4,5]
1 3 [2,3,4,5]
2 5 [2,4,5]
[2,4]
You can iterate over a different list than you are modifying. This returns a copy of the list
def cut_uniq(data):
return [x for x in data if data.count(x) > 1]
or more efficiently
from collection import Counter
def cut_uniq(data):
return [x for x, count in Counter(data) if count > 1]
If you really do want to modify the original list, and not return a copy
def cut_uniq(data):
i = 0
while i < len(data):
if data.count(data[i]) == 1:
del data[i]
else:
i += 1
or
from collections import Counter
def cut_uniq(data):
for x, count in Counter(data):
if count == 1:
data.remove(x)
95% of the time that you modify the same list as you're iterating over, you'll have problems.
When you use
for x in data:
it translates to
for i in [0,1,2,3,4]:
x = data[i]
So in the first loop, i = 0 data[i]=1. you remove 1 from data, the data is [2,3,4,5]
on the second loop , i = 1, because now data is [2,3,4,5], data[i] = 3. So 2 is left in the data list and never been visited.
Same as the number 4.
So when you finished your loop, the [2,4] leted in the list.
Suppose I have a QuerySet that returns 10 objects, 3 of which will be displayed in the following positions:
[ display 1 position ] [ display 2 position ] [ display 3 position ]
The model representing it is as follows:
class FeaturedContent(models.Model):
image = models.URLField()
position = models.PositiveSmallIntegerField(blank=True, null=True)
where position can be either 1, 2, 3, or unspecified (Null).
I want to be able to order the QuerySet randomly EXCEPT FOR the objects with a specified position. However, I can't order it by doing:
featured_content = FeaturedContent.objects.order_by('-position', '?')
Because if I had one item that had position = 2, and all the other items were Null, then the item would appear in position 1 instead of position 2.
How would I do this ordering?
Thinking about this, perhaps it would be best to have the data as a dict instead of a list, something like:
`{'1': item or null, '2': item or null, '3': item or null, '?': [list of other items]}`
If you use a db backend that does random ordering efficiently you could do it like this:
# This will hold the result
featured_dict = {}
featured_pos = FeaturedContent.objects.filter(position__isnull=False).order_by('-position')
featured_rand = FeaturedContent.objects.filter(position__isnull=True).order_by('?')
pos_index = 0
rand_index = 0
for pos in range(1, 4):
content = None
if pos_index < len(featured_pos) and featured_pos[pos_index].position == pos:
content = featured_pos[pos_index]
pos_index += 1
elif rand_index < len(featured_rand):
content = featured_rand[rand_index]
rand_index += 1
featured_dict[str(pos)] = content
# I'm not sure if you have to check for a valid index first before slicing
featured_dict['?'] = featured_rand[rand_index:]
If you just want to iterate over the queryset you can have two querysets, order them and chain them.
import itertools
qs1 = FeaturedContent.objects.filter(position__isnull=False).order_by('-position')
qs2 = FeaturedContent.objects.filter(position__isnull=True).order_by('?')
featured_content = itertools.chain(qs1, qs2)
for item in featured_content:
#do something with qs item
print item
Upadate:
Since you are asking to make sure position determines the order and the "blank" spaces are replaced randomly by elements with null positions. If the featured list you want to obtain is not too large, 20 in this case
featured = []
rands = []
for i in xrange(1, 20):
try:
x = FeaturedContent.objects.get(position=i) # assuming position is unique
except FeaturedContentDoesNotExist:
if not rands:
rands = list(FeaturedContent.objects.filter(position__isnull=True).order_by('?')[:20]
x = rands[0]
rands = rands[1:]
featured.append(x)
I would post process it, doing a merge sort between the ordered and unordered records.
EDIT:
The beginnings of a generator for this:
def posgen(posseq, arbseq, posattr='position', startpos=1):
posel = next(posseq)
for cur in itertools.count(startpos):
if getattr(posel, posattr) == cur:
yield posel
posel = next(posseq)
else:
yield next(arbseq)
Note that there are lots of error conditions possible in this code (hint: StopIteration).
heres a quick one for you:
I have a list of id's which I want to use to return a QuerySet(or array if need be), but I want to maintain that order.
Thanks
Since Django 1.8, you can do:
from django.db.models import Case, When
pk_list = [10, 2, 1]
preserved = Case(*[When(pk=pk, then=pos) for pos, pk in enumerate(pk_list)])
queryset = MyModel.objects.filter(pk__in=pk_list).order_by(preserved)
I don't think you can enforce that particular order on the database level, so you need to do it in python instead.
id_list = [1, 5, 7]
objects = Foo.objects.filter(id__in=id_list)
objects = dict([(obj.id, obj) for obj in objects])
sorted_objects = [objects[id] for id in id_list]
This builds up a dictionary of the objects with their id as key, so they can be retrieved easily when building up the sorted list.
If you want to do this using in_bulk, you actually need to merge the two answers above:
id_list = [1, 5, 7]
objects = Foo.objects.in_bulk(id_list)
sorted_objects = [objects[id] for id in id_list]
Otherwise the result will be a dictionary rather than a specifically ordered list.
Here's a way to do it at database level. Copy paste from: blog.mathieu-leplatre.info
:
MySQL:
SELECT *
FROM theme
ORDER BY FIELD(`id`, 10, 2, 1);
Same with Django:
pk_list = [10, 2, 1]
ordering = 'FIELD(`id`, %s)' % ','.join(str(id) for id in pk_list)
queryset = Theme.objects.filter(pk__in=[pk_list]).extra(
select={'ordering': ordering}, order_by=('ordering',))
PostgreSQL:
SELECT *
FROM theme
ORDER BY
CASE
WHEN id=10 THEN 0
WHEN id=2 THEN 1
WHEN id=1 THEN 2
END;
Same with Django:
pk_list = [10, 2, 1]
clauses = ' '.join(['WHEN id=%s THEN %s' % (pk, i) for i, pk in enumerate(pk_list)])
ordering = 'CASE %s END' % clauses
queryset = Theme.objects.filter(pk__in=pk_list).extra(
select={'ordering': ordering}, order_by=('ordering',))
id_list = [1, 5, 7]
objects = Foo.objects.filter(id__in=id_list)
sorted(objects, key=lambda i: id_list.index(i.pk))
Another better/cleaner approach can be
pk_list = [10, 2, 1]
sorted_key_object_pair = MyModel.objects.in_bulk(pk_list)
sorted_objects = sorted_key_object_pair.values()
Simple, clean, less code.
I have a list of integers which I need to parse into a string of ranges.
For example:
[0, 1, 2, 3] -> "0-3"
[0, 1, 2, 4, 8] -> "0-2,4,8"
And so on.
I'm still learning more pythonic ways of handling lists, and this one is a bit difficult for me. My latest thought was to create a list of lists which keeps track of paired numbers:
[ [0, 3], [4, 4], [5, 9], [20, 20] ]
I could then iterate across this structure, printing each sub-list as either a range, or a single value.
I don't like doing this in two iterations, but I can't seem to keep track of each number within each iteration. My thought would be to do something like this:
Here's my most recent attempt. It works, but I'm not fully satisfied; I keep thinking there's a more elegant solution which completely escapes me. The string-handling iteration isn't the nicest, I know -- it's pretty early in the morning for me :)
def createRangeString(zones):
rangeIdx = 0
ranges = [[zones[0], zones[0]]]
for zone in list(zones):
if ranges[rangeIdx][1] in (zone, zone-1):
ranges[rangeIdx][1] = zone
else:
ranges.append([zone, zone])
rangeIdx += 1
rangeStr = ""
for range in ranges:
if range[0] != range[1]:
rangeStr = "%s,%d-%d" % (rangeStr, range[0], range[1])
else:
rangeStr = "%s,%d" % (rangeStr, range[0])
return rangeStr[1:]
Is there a straightforward way I can merge this into a single iteration? What else could I do to make it more Pythonic?
>>> from itertools import count, groupby
>>> L=[1, 2, 3, 4, 6, 7, 8, 9, 12, 13, 19, 20, 22, 23, 40, 44]
>>> G=(list(x) for _,x in groupby(L, lambda x,c=count(): next(c)-x))
>>> print ",".join("-".join(map(str,(g[0],g[-1])[:len(g)])) for g in G)
1-4,6-9,12-13,19-20,22-23,40,44
The idea here is to pair each element with count(). Then the difference between the value and count() is constant for consecutive values. groupby() does the rest of the work
As Jeff suggests, an alternative to count() is to use enumerate(). This adds some extra cruft that needs to be stripped out in the print statement
G=(list(x) for _,x in groupby(enumerate(L), lambda (i,x):i-x))
print ",".join("-".join(map(str,(g[0][1],g[-1][1])[:len(g)])) for g in G)
Update: for the sample list given here, the version with enumerate runs about 5% slower than the version using count() on my computer
Whether this is pythonic is up for debate. But it is very compact. The real meat is in the Rangify() function. There's still room for improvement if you want efficiency or Pythonism.
def CreateRangeString(zones):
#assuming sorted and distinct
deltas = [a-b for a, b in zip(zones[1:], zones[:-1])]
deltas.append(-1)
def Rangify((b, p), (z, d)):
if p is not None:
if d == 1: return (b, p)
b.append('%d-%d'%(p,z))
return (b, None)
else:
if d == 1: return (b, z)
b.append(str(z))
return (b, None)
return ','.join(reduce(Rangify, zip(zones, deltas), ([], None))[0])
To describe the parameters:
deltas is the distance to the next value (inspired from an answer here on SO)
Rangify() does the reduction on these parameters
b - base or accumulator
p - previous start range
z - zone number
d - delta
To concatenate strings you should use ','.join. This removes the 2nd loop.
def createRangeString(zones):
rangeIdx = 0
ranges = [[zones[0], zones[0]]]
for zone in list(zones):
if ranges[rangeIdx][1] in (zone, zone-1):
ranges[rangeIdx][1] = zone
else:
ranges.append([zone, zone])
rangeIdx += 1
return ','.join(
map(
lambda p: '%s-%s'%tuple(p) if p[0] != p[1] else str(p[0]),
ranges
)
)
Although I prefer a more generic approach:
from itertools import groupby
# auxiliary functor to allow groupby to compare by adjacent elements.
class cmp_to_groupby_key(object):
def __init__(self, f):
self.f = f
self.uninitialized = True
def __call__(self, newv):
if self.uninitialized or not self.f(self.oldv, newv):
self.curkey = newv
self.uninitialized = False
self.oldv = newv
return self.curkey
# returns the first and last element of an iterable with O(1) memory.
def first_and_last(iterable):
first = next(iterable)
last = first
for i in iterable:
last = i
return (first, last)
# convert groups into list of range strings
def create_range_string_from_groups(groups):
for _, g in groups:
first, last = first_and_last(g)
if first != last:
yield "{0}-{1}".format(first, last)
else:
yield str(first)
def create_range_string(zones):
groups = groupby(zones, cmp_to_groupby_key(lambda a,b: b-a<=1))
return ','.join(create_range_string_from_groups(groups))
assert create_range_string([0,1,2,3]) == '0-3'
assert create_range_string([0, 1, 2, 4, 8]) == '0-2,4,8'
assert create_range_string([1,2,3,4,6,7,8,9,12,13,19,20,22,22,22,23,40,44]) == '1-4,6-9,12-13,19-20,22-23,40,44'
This is more verbose, mainly because I have used generic functions that I have and that are minor variations of itertools functions and recipes:
from itertools import tee, izip_longest
def pairwise_longest(iterable):
"variation of pairwise in http://docs.python.org/library/itertools.html#recipes"
a, b = tee(iterable)
next(b, None)
return izip_longest(a, b)
def takeuntil(predicate, iterable):
"""returns all elements before and including the one for which the predicate is true
variation of http://docs.python.org/library/itertools.html#itertools.takewhile"""
for x in iterable:
yield x
if predicate(x):
break
def get_range(it):
"gets a range from a pairwise iterator"
rng = list(takeuntil(lambda (a,b): (b is None) or (b-a>1), it))
if rng:
b, e = rng[0][0], rng[-1][0]
return "%d-%d" % (b,e) if b != e else "%d" % b
def create_ranges(zones):
it = pairwise_longest(zones)
return ",".join(iter(lambda:get_range(it),None))
k=[0,1,2,4,5,7,9,12,13,14,15]
print create_ranges(k) #0-2,4-5,7,9,12-15
def createRangeString(zones):
"""Create a string with integer ranges in the format of '%d-%d'
>>> createRangeString([0, 1, 2, 4, 8])
"0-2,4,8"
>>> createRangeString([1,2,3,4,6,7,8,9,12,13,19,20,22,22,22,23,40,44])
"1-4,6-9,12-13,19-20,22-23,40,44"
"""
buffer = []
try:
st = ed = zones[0]
for i in zones[1:]:
delta = i - ed
if delta == 1: ed = i
elif not (delta == 0):
buffer.append((st, ed))
st = ed = i
else: buffer.append((st, ed))
except IndexError:
pass
return ','.join(
"%d" % st if st==ed else "%d-%d" % (st, ed)
for st, ed in buffer)
Here is my solution. You need to keep track of various pieces of information while you iterate through the list and create the result - this screams generator to me. So here goes:
def rangeStr(start, end):
'''convert two integers into a range start-end, or a single value if they are the same'''
return str(start) if start == end else "%s-%s" %(start, end)
def makeRange(seq):
'''take a sequence of ints and return a sequence
of strings with the ranges
'''
# make sure that seq is an iterator
seq = iter(seq)
start = seq.next()
current = start
for val in seq:
current += 1
if val != current:
yield rangeStr(start, current-1)
start = current = val
# make sure the last range is included in the output
yield rangeStr(start, current)
def stringifyRanges(seq):
return ','.join(makeRange(seq))
>>> l = [1,2,3, 7,8,9, 11, 20,21,22,23]
>>> l2 = [1,2,3, 7,8,9, 11, 20,21,22,23, 30]
>>> stringifyRanges(l)
'1-3,7-9,11,20-23'
>>> stringifyRanges(l2)
'1-3,7-9,11,20-23,30'
My version will work correctly if given an empty list, which I think some of the others will not.
>>> stringifyRanges( [] )
''
makeRanges will work on any iterator that returns integers and lazily returns a sequence of strings so can be used on infinite sequences.
edit: I have updated the code to handle single numbers that are not part of a range.
edit2: refactored out rangeStr to remove duplication.
how about this mess...
def rangefy(mylist):
mylist, mystr, start = mylist + [None], "", 0
for i, v in enumerate(mylist[:-1]):
if mylist[i+1] != v + 1:
mystr += ["%d,"%v,"%d-%d,"%(start,v)][start!=v]
start = mylist[i+1]
return mystr[:-1]