Related
I'm trying to replicate numpy.tensordot in c++. The example in the numpy documentation shows a nested loop that I can get to work, but what if instead of
c = np.tensordot(a,b, axes=([1,0],[0,1]))
I want to do:
c = np.tensordot(a,b, axes=([1,2],[0,1]))
What would that new nested loop look like in python? And is there an easier/faster way of doing this operation in c++? Right now I'm using the same nested "for" loops with std::vector's in c++. I've seen a few libraries that might help, but I'm trying to use just the c++ standard library.
Here is that numpy example, and the link to the documentation: https://numpy.org/doc/stable/reference/generated/numpy.tensordot.html
Examples
A “traditional” example:
>>>
a = np.arange(60.).reshape(3,4,5)
b = np.arange(24.).reshape(4,3,2)
c = np.tensordot(a,b, axes=([1,0],[0,1]))
c.shape
(5, 2)
c
array([[4400., 4730.],
[4532., 4874.],
[4664., 5018.],
[4796., 5162.],
[4928., 5306.]])
# A slower but equivalent way of computing the same...
d = np.zeros((5,2))
for i in range(5):
for j in range(2):
for k in range(3):
for n in range(4):
d[i,j] += a[k,n,i] * b[n,k,j]
c == d
array([[ True, True],
[ True, True],
[ True, True],
[ True, True],
[ True, True]])
Thank you
I find rewriting to np.einsum first is helpful as the resulting for loop code looks quite similar conceptually:
a = np.random.rand(16, 8, 2)
b = np.random.rand(8, 2, 1)
c = np.tensordot(a, b, axes=([1,2],[0,1]))
# same thing written with einsum
c_ein = np.einsum("ijk,jko->io", a, b)
# same thing done with for loops,
# notice how we can use the same letters and indexing as einsum
c_manual = np.zeros((16, 1))
for i in range(16):
for o in range(1):
# j and k are summed since they don't appear in output
total = 0
for j in range(8):
for k in range(2):
total += a[i, j, k] * b[j, k, o]
c_manual[i, o] = total
assert np.allclose(c, c_ein, c_manual)
import tensorflow as tf
cluster_size = tf.constant(6) # size of the cluster
m = tf.constant(6) # number of contigs (column size)
n = tf.constant(3) # number of points in a single contigs (column size)
contigs_index = tf.reshape(tf.range(0, m, 1, dtype=tf.int32), [1, -1])
contigs = tf.constant(
[[1.1, 2.2, 3.3], [6.6, 5.5, 4.4], [7.7, 8.8, 9.9], [11.1, 22.2, 33.3],
[66.6, 55.5, 44.4], [77.7, 88.8, 99.9]])
# pad zeo to the right till fixed length
def rpad_with_zero(points):
points = tf.slice(tf.pad(points, tf.reshape(tf.concat(
[tf.zeros([1, 2], tf.int32), tf.add(
tf.zeros([1, 2], tf.int32),
tf.subtract(cluster_size, tf.size(points)))], 0), [2, -1]), "CONSTANT"),
(0, tf.subtract(cluster_size, tf.size(points))),
(1, cluster_size))
return points
#calculate pearson correlation coefficient r value
def calculate_pcc(row, contigs):
r = tf.divide(tf.subtract(
tf.multiply(tf.to_float(n), tf.reduce_sum(tf.multiply(row, contigs), 1)),
tf.multiply(tf.reduce_sum(row, 1), tf.reduce_sum(contigs, 1))),
tf.multiply(
tf.sqrt(tf.subtract(
tf.multiply(tf.to_float(n), tf.reduce_sum(tf.square(row), 1)),
tf.square(tf.reduce_sum(row, 1)))),
tf.sqrt(tf.subtract(tf.multiply(
tf.to_float(n), tf.reduce_sum(tf.square(contigs), 1)),
tf.square(tf.reduce_sum(contigs, 1)))
)))
return r
#slice first row from contigs
row = tf.slice(contigs, (0, 0), (1, 3))
#calculate pcc
r = calculate_pcc(row, contigs)
#cluster member index whose r value is greater than 0.90, then casting to
# int32,
members0_index = tf.cast(tf.reshape(tf.where(tf.greater(r, 0.90)), [1, -1]),
tf.int32)
#members = index <intersection> members, padding the members index with
# zeros at right, to keep the fixed cluster length
members0_index = rpad_with_zero(
tf.reshape(tf.sets.set_intersection(contigs_index, members0_index).values,
[1, -1]))
#update index with the rest element index from contigs, and padding
contigs_index = rpad_with_zero(
tf.reshape(tf.sets.set_difference(contigs_index, members0_index).values,
[1, -1]))
#def condition(contigs, contigs_index, members0_index):
def condition(contigs_index, members0_index):
return tf.greater(tf.count_nonzero(contigs_index),
0) # iterate until there is a contig
#def body(contigs, contigs_index, members0_index):
def body(contigs_index, members0_index):
i = tf.reshape(tf.slice(contigs_index, [0, 0], [1, 1]),
[]) #the first element in the contigs_index
row = tf.slice(contigs, (i, 0),
(1, 3)) #slice the ith contig from contigs
r = calculate_pcc(row, contigs)
members_index = tf.cast(tf.reshape(tf.where(tf.greater(r, 0.90)), [1, -1]),
tf.int32)
members_index = rpad_with_zero(rpad_with_zero(
tf.reshape(tf.sets.set_intersection(contigs_index, members_index).values,
[1, -1])))
members0_index = tf.concat([members0_index, members_index], 0)
contigs_index = rpad_with_zero(
tf.reshape(tf.sets.set_difference(contigs_index, members_index).values,
[1, -1]))
#return [contigs, contigs_index, members0_index]
return [contigs_index, members0_index]
sess = tf.Session()
sess.run(tf.while_loop(condition, body,
#loop_vars=[contigs, contigs_index, members0_index],
loop_vars=[contigs_index, members0_index],
#shape_invariants=[contigs.get_shape(), contigs_index.get_shape(),
# tf.TensorShape([None, 6])]))
shape_invariants=[contigs_index.get_shape(), tf.TensorShape([None, 6])]))
The error is:
ValueError: The shape for while_12/Merge:0 is not an invariant for the
loop. It enters the loop with shape (1, 6), but has shape (?, ?) after
one iteration. Provide shape invariants using either the
shape_invariants argument of tf.while_loop or set_shape() on the
loop variables.
It seems the variable
contigs_index
is responsible, but i really don't know why! I unfold the loop execute each statement but could not find any shape mismatch!
shape_invariants=[contigs_index.get_shape(), tf.TensorShape([None, 6])])) should become shape_invariants=[tf.TensorShape([None, None]), tf.TensorShape([None, 6])])), to allow for shape changes of contigs_index variable (in the rpad_with_zero call).
i'm trying to teach myself some basic coding by recreating 2048, but i've hit a wall.
what i'm trying to do is:
1) create a matrix representing the spaces on the board, occupied by values of zero for empty spaces
2) transpose the axes of the matrix according to the axis the player wants to move the pieces in
3) create a list from each row in the matrix using np.tolist()
4) use listID.sort(key=bool) to sort the list by its truth value, placing the zeroes at one end of the list without rearranging the items that have a value, and saving space by modifying the lists in-place
5) merge adjacent items in the lists that are equal in value
6) use np.columnstack to combine the sorted lists into a matrix, ovewriting the previous matrix
7) transpose the axes to what they originally were
where i'm getting tripped up is at step 4).
the lists don't get sorted at all, nothing changes.
for example a0 (the first row's list) will be [0, 2, 0, 0], but after sorting it, it's STILL [0, 2, 0, 0].
here's my awful, nooby, WIP code to look at and look for yourself if you so desire:
import numpy as np
a = np.matrix([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]])
print (a)
b = np.random.randint(low=0, high=3)
c = b
while c == b:
c = np.random.randint(low=0, high=3)
d = np.random.randint(low=0, high=3)
e = d
while e == d:
e = np.random.randint(low=0, high=3)
a.itemset((b, c), 2)
a.itemset((d, e), 2)
print (a)
score = np.sum(a)
print (score)
#while 0 in a:
#direction = input("U, D, L, R?")
#if direction == "U":
# mode = 1
#if direction == "D":
# mode = 2
#if direction == "L":
# mode = 3
#if direction == "R":
# mode = 4
#transpose axes according to mode
#sort each array in matrix by boolean value
#refuses to sort?
a0 = a[0].tolist()
a0.sort(key=bool)
a1 = a[1].tolist()
a1.sort(key=bool)
a2 = a[2].tolist()
a2.sort(key=bool)
a3 = a[3].tolist()
a3.sort(key=bool)
#reverse each list depending on mode
#combine back into matrix
print (a0, a1, a2, a3)
a = np.column_stack([[a0], [a1], [a2], [a3]])
print (a)
i've searched and searched but nothing i've seen has offered some insight into how i'm doing this wrong, which i'd presume that i am.
edit: SOLVED!
import numpy as np
import numpy.matlib
import itertools
a = np.matlib.zeros((4,4))
print (a)
b = np.random.randint(low=0, high=3)
c = b
while c == b:
c = np.random.randint(low=0, high=3)
d = np.random.randint(low=0, high=3)
e = d
while e == d:
e = np.random.randint(low=0, high=3)
a.itemset((b, c), float(2))
a.itemset((d, e), float(2))
print (a)
score = np.sum(a)
print (score)
#while 1 in a:
#direction = input("U, D, L, R?")
#if direction == "U":
# mode = 1
#if direction == "D":
# mode = 2
#if direction == "L":
# mode = 3
#if direction == "R":
# mode = 4
#transpose axes according to mode
#sort each array in matrix by boolean value
#refuses to sort?
a0 = a[0].tolist()
a01 = []
a0_2 = list(itertools.chain.from_iterable(a0))
for item in a0_2:
a01.append(float(item))
a01.sort(key=bool)
a1 = a[1].tolist()
a11 = []
a1_2 = list(itertools.chain.from_iterable(a1))
for item in a1_2:
a11.append(float(item))
a11.sort(key=bool)
a2 = a[2].tolist()
a21 = []
a2_2 = list(itertools.chain.from_iterable(a2))
for item in a2_2:
a21.append(float(item))
a21.sort(key=bool)
a3 = a[3].tolist()
a31 = []
a3_2 = list(itertools.chain.from_iterable(a3))
for item in a3_2:
a31.append(float(item))
a31.sort(key=bool)
#reverse each list depending on mode
#combine back into matrix
print (a01, a11, a21, a31)
a = np.vstack([[a01], [a11], [a21], [a31]])
print (a)
You say you're trying to "sort by truth values" and that NumPy doesn't have an option for that. Well, it does, sort of!
a0 = a0[(~a0.astype(bool)).argsort(kind='mergesort')]
That takes a0 and converts it to True wherever it was nonzero, then inverts that (so False for nonzero), then sorts it (using mergesort for stable sorting). The use of argsort() lets us do "indirect" sorting, i.e. sort one thing by another.
The way you have done it by converting the arrays into lists and then back again is quite inefficient by contrast. But you won't notice this if the data are small, as they are in your case.
My code is currently written as:
convert = {0:0,1:1,2:2,3:3,4:0,5:1,6:2,7:1}
rows = [[convert[random.randint(0,7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - rows.count(0)
print numgood
>> 25
It always comes out as 25, so it's not just that rows contains no 0's.
Have you printed rows?
It's [[0, 1, 0, 0, 2], [1, 2, 0, 1, 2], [3, 1, 1, 1, 1], [1, 0, 0, 1, 0], [0, 3, 2, 0, 1]], so you have a nested list there.
If you want to count the number of 0's in those nested lists, you could try:
import random
convert = {0:0, 1:1, 2:2, 3:3, 4:0, 5:1, 6:2, 7:1}
rows = [[convert[random.randint(0, 7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - sum(e.count(0) for e in rows)
print numgood
Output:
18
rows doesn't contain any zeroes; it contains lists, not integers.
>>> row = [1,2,3]
>>> type(row)
<type 'list'>
>>> row.count(2)
1
>>> rows = [[1,2,3],[4,5,6]]
>>> rows.count(2)
0
>>> rows.count([1,2,3])
1
To count the number of zeroes in any of the lists in rows, you could use a generator expression:
>>> rows = [[1,2,3],[4,5,6], [0,0,8]]
>>> sum(x == 0 for row in rows for x in row)
2
You could also use numpy:
import numpy as np
import random
convert = {0:0,1:1,2:2,3:3,4:0,5:1,6:2,7:1}
rows = [[convert[random.randint(0,7)] for _ in range(5)] for _ in range(5)]
numgood = 25 - np.count_nonzero(rows)
print numgood
Output:
9
I need help with a function that can return words that have 3 or more characters that are "evenly" spaced, that is the ord() value for consecutive letters left to right are even (same difference value). This is what I have so far... and the output is this:
test_list2 = ['i', 'made', 'an', 'ace', 'today', 'at', 'tennis...yeaah', 'booi', ':d']
for word in test_list2:
if len(word) >=3:
temp_list = []
for chr1 in word:
if word.index(chr1) != (len(word)-1):
chr2 = word.index(chr1)+1
num = ord(word[chr2]) - ord(chr1)
temp_list.append(num)
temp_tup = (word, temp_list)
final_list.append(temp_tup)
final_list = [('made', [-12, 3, 1]), ('ace', [2, 2]), ('today', [-5, -11, -3, 24]),
('tennis...yeaah', [-15, 9, 0, 0, 10, -69, 0, 0, 0, -20, 9, 0, 0]),
('booi', [13, 0, 0])]
But i need to return only the ones that are evenly spaced ('ace'). The output should be like this,
[('ace',2)]
Assuming that you do not need the final_list with non evenly spaced numbers, then you can keep track of the num to see if it stays the same throughout the word. If you find a different num stop and go to the next word. If num stays the same then add a (word, num) tuple to the final_list:
for word in test_list2:
if len(word) >=3:
all_nums_are_same = True
prev_num = None
for chr1 in word:
if word.index(chr1) != (len(word)-1):
chr2 = word.index(chr1)+1
num = ord(word[chr2]) - ord(chr1)
if not prev_num:
prev_num = num
elif prev_num != num:
# different number is found, we can
# stop and move on to next word
all_nums_are_same = False
break
if all_nums_are_same:
# only add tuple if all numbers we the same
temp_tup = (word, prev_num)
final_list.append(temp_tup)
This yields [('ace',2)] as a result.
I banged this out in Python 3.3, compiles and works on my machine :)
There's a bunch of extra debugging junk in there like print statements, if you want to test it with some more complicated data (example: long blocks of text) for bugs.
I made use of enumerate(), rather than your word.index, not sure which is more pythonic?
import sys
### Define variables
test_list = ['i', 'made', 'an', 'ace', 'today', 'at', 'tennis...yeaah', 'booi', ':d']
proc_check = [('made', [-12, 3, 1]),
('ace', [2, 2]),
('today', [-5, -11, -3, 24]),
('tennis...yeaah', [-15, 9, 0, 0, 10, -69, 0, 0, 0, -20, 9, 0, 0]),
('booi', [13, 0, 0])]
final_check = [('ace', [2,2])]
test_list2 = ['ace', 'ace', 'ace']
proc_check2 = [('ace', [2, 2]),
('poo', [3, 3]),
('ace', [2, 2])]
final_check2 = [('ace', [2,2]),('poo', [2,2]),('ace', [2,2])]
### Function definitions
def wordIsEven(word_list, proc_list_check):
final_list = []
procd_list = []
for word in word_list:
temp_list = []
if len(word) >= 3:
for chr1 in word:
if word.index(chr1) != (len(word)-1):
chr2 = word.index(chr1)+1
num = ord(word[chr2]) - ord(chr1)
temp_list.append(num)
temp_tup = (word, temp_list)
procd_list.append(temp_tup)
errors = False
for i, check in enumerate(procd_list):
if check != proc_list_check[i]:
errors = True
print("Word Eval Fail! " + str(check) + " != " + str(proc_list_check[i]))
if errors == True:
print("List compare failed!" )
else:
print("Lists compare equally!")
for tmp_tup in procd_list:
print("Examining Slice: "+str(tmp_tup[1]))
for i, num in enumerate(tmp_tup[1]):
if i + 1 < len(tmp_tup[1]):
num2 = tmp_tup[1][i+1]
if num == num2:
if num != 0:
print("Got one! " + str(tmp_tup))
final_list.append(tmp_tup)
return final_list
### Code execution
my_list = wordIsEven(test_list2, proc_check2)
my_check = final_check2
print("Printing Final list:")
for i, item in enumerate(my_list):
tempStr = str(item)
if item != my_check[i]:
tempStr += " doesn't match check data..." + str(my_check[i])
print(tempStr)
sys.exit()