I'm trying to use a string as a parameter in a function. Is something like this possible?
from django.contrib.auth.models import User
param_var = 'contains'
User.objects.filter(first_name__{param_var}='John')
# ideally, this should equate to:
User.objects.filter(first_name__contains='John')
param_var = 'contains'
User.objects.filter(**{'first_name__%s'%param_var: 'John'})
Though I really wonder about the wisdom of doing that... I think you may be re-inventing sql injection...
I suspect there's a better way. You should probably explain more of what you are trying to accomplish.
Solved:
# Something like this...
qs = User.objects.all()
parameters = {
'name': {
'type': 'get',
'filter': [
'first_name__icontains',
'last_name__icontains',
# ...
],
},
# ...
}
for parameter, data in parameters.items():
if data['type'] == 'get':
value = request.GET.get(parameter, None)
elif data['type'] == 'getlist':
value = request.GET.getlist(parameter)
if value:
query = Q()
for f in data['filter']:
query = query | Q(**{ f : value })
qs = qs.filter(query)
Related
The issue
I'm trying out great expectations with dagster, as per this guide
My pipeline seems to execute correctly until it reaches this block:
expectation = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='dev.data-pipeline-data-storage.data_pipelines.raw_data.sirene_update',
suite_name='suite.data_pipelines.raw_data.sirene_update',
)
if expectation["success"]:
print("Success")
trying to call expectation["success"] results in a
# TypeError: 'SolidDefinition' object is not subscriptable
When I go inside the code of ge_validation_op_factory, there is a _ge_validation_fn that should yield ExpectationResult, but somehow it gets coverted into a SolidDefinition...
Dagster version = 0.15.9;
Great Expectations version = 0.15.44
Code to reproduce the error
In my code, I am trying to interact with an s3 bucket, so it would be a bit tedious to re-create the code for my example but here it is anyway:
In a gx_postprocessing.py
import json
import boto3
import dagster_ge
from dagster import (
op,
graph,
Field,
String,
OpExecutionContext,
)
from typing import List, Dict
#op(
config_schema={
"bucket": Field(
String,
description="s3 bucket name",
),
"path_in_s3": Field(
String,
description="Prefix representing the path to data",
),
"technical_date": Field(
String,
description="date string to fetch data",
),
"file_name": Field(
String,
description="file name that contains the data",
),
}
)
def read_in_json_datafile_from_s3(context: OpExecutionContext):
bucket = context.op_config["bucket"]
path_in_s3 = context.op_config["path_in_s3"]
technical_date = context.op_config["technical_date"]
file_name = context.op_config["file_name"]
object = f"{path_in_s3}/" f"technical_date={technical_date}/" f"{file_name}"
s3 = boto3.resource("s3")
content_object = s3.Object(bucket, object)
file_content = content_object.get()["Body"].read().decode("utf-8")
json_content = json.loads(file_content)
return json_content
#op
def process_example_dq(data: List[Dict]):
return len(data)
#op
def postprocess_example_dq(numrows, expectation):
if expectation["success"]:
return numrows
else:
raise ValueError
#op
def validate_example_dq(context: OpExecutionContext):
expectation = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='my_bucket.data_pipelines.raw_data.example_update',
suite_name='suite.data_pipelines.raw_data.example_update',
)
return expectation
#graph(
config={
"read_in_json_datafile_from_s3": {
"config": {
"bucket": "my_bucket",
"path_in_s3": "my_path",
"technical_date": "2023-01-24",
"file_name": "myfile_20230124.json",
}
},
},
)
def example_update_evaluation():
output_dict = read_in_json_datafile_from_s3()
nb_items = process_example_dq(data=output_dict)
expectation = validate_example_dq()
postprocess_example_dq(
numrows=nb_items,
expectation=expectation,
)
Do not forget to add great_expectations_poc_pipeline to your __init__.py where the pipelines=[..] are listed.
In this example, dagster_ge.ge_validation_op_factory(...) is returning an OpDefinition, which is the same type of thing as (for example) process_example_dq, and should be composed in the graph definition the same way, rather than invoked within another op.
So instead, you'd want to have something like:
validate_example_dq = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='my_bucket.data_pipelines.raw_data.example_update',
suite_name='suite.data_pipelines.raw_data.example_update',
)
Then use that op inside your graph definition the same way you currently are (i.e. expectation = validate_example_dq())
I am trying to return most voted answer for each question.also i want to send also extra infomation of that answer like vote and id.
Printing one value is easy but for more than one i have to return dictionary.So how can i return dictionary and print all values in template.
from django import template
register = template.Library()
#register.simple_tag
def getmostvotedanswer(answers):
answer = answers.order_by('-vote')[0]
answer_info = {
'answer':answer.answer,
'vote':answer.vote,
'id':answer.id
}
return answer_info
index.html
<p class="small text-muted ">{% getmostvotedanswer question.answer_set.all %}</p>
Output
{'answer': 'THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER', 'vote': 7, 'id': 1}
I can call template_tag 3 times for three values.
But I don't want to call templatetag again and again i think it will affect performance.
view.py
def index(request):
questions = Question.objects.all()
context = {
'questions':questions
}
return render(request,'index.html',context=context)
Edit -> Add view.py
The easiest, debugable, best performance and test friendly way to achieve it is to cook data on view instead of write custom template tag. Windows functions are needed to get the first answer of each question:
from django.db.models import F, Window
from django.db.models.functions.window import FirstValue
def index(request):
#q_and_a_ids = [ (id question, id most voted answer), (... ]
q_and_a_ids = (
Question
.objects
.annotate(
most_voted_id=Window(
expression=FirstValue('answer__id'),
partition_by=['id'],
order_by=F('answer__vote').desc()
)
)
.distinct()
.values_list( 'id', 'most_voted_id')
)
answers_ids = set( [ a_id for (_,a_id) in q_and_a_ids] )
questions_dict = Question.objects.in_bulk()
answers_dict = Answers.objects.filter(pk__in=answers_ids).in_bulk()
#q_and_a = [ { 'q':question, 'a':most voted answer}, { ... ]
q_and_a = [ {'q': questions_dict[q_id],
'a': answers_dict.get(a_id) }
for (q_id,a_id) in q_and_a_ids ]
context = {
'questions_and_answers': q_and_a
}
return render(request,'index.html',context=context)
I am having a problem when I get the value from a hstore field. The situation is something like this:
model:
class Model(models.Model):
hstoreField = HStoreField()
test cases:
def test_case_1(self):
new_model = Model.objects.create(
hstoreField = {
'key1': 'value1',
'key2': 'value2'
}
)
self.assertEqual(new_model.subscription['key1'], 'value1')
self.assertEqual(new_model.subscription['key2'], 'value2')
def test_case_2(self):
Model.objects.create(
hstoreField = {
'key1': 'value1',
'key2': 'value2'
}
)
new_model = Model.objects.get( id = 1 ) // assume it works
self.assertEqual(new_model.hstoreField['key1'], 'value1')
self.assertEqual(new_model.hstoreField['key2'], 'value2')
When I run those test cases, the first one passes, but the second one fails and the error thrown is: TypeError: string indices must be integers. Checking the value of hstoreField is a string and it should be a dict.
The error only happens in tests, in real code both cases work. Can you help me guys?
In tastypie, I want set json result name.
I have a class that I use for it but I can set name in.
enter cclass ContentResource(ModelResource):
class Meta:
results = ListField(attribute='results')
queryset = Content.objects.all()
resource_name = 'content'
max_limit = None
#filtering = {"title": "contains"}
def alter_list_data_to_serialize(self, request, data_dict):
if isinstance(data_dict, dict):
if 'meta' in data_dict:
# Get rid of the "meta".
del(data_dict['meta'])
# Rename the objects.
data_dict['Mobile'] = data_dict['objects']
del(data_dict['objects'])
return data_dict
ode here it returns this
{"Mobile":
[
{
"added": "2015-07-23T11:30:20.911835",
"content_cast": "",
"content_company": "HamrahCinema",
"content_description": "so nice",
"content_director": "",
"content_duration": "2:20",
"content_filelanguage": null,
}
]
}
when I use /content/api/content every thing is ok, but when I use /content/api/content/1,"mobile" is removed.
as educated guess, I would suggest using alter_detail_data_to_serialize
So I am trying to mock all the stripe web hooks in the method so that I can write the Unit test for it. I am using the mock library for mocking the stripe methods. Here is the method I am trying to mock:
class AddCardView(APIView):
"""
* Add card for the customer
"""
permission_classes = (
CustomerPermission,
)
def post(self, request, format=None):
name = request.DATA.get('name', None)
cvc = request.DATA.get('cvc', None)
number = request.DATA.get('number', None)
expiry = request.DATA.get('expiry', None)
expiry_month, expiry_year = expiry.split("/")
customer_obj = request.user.contact.business.customer
customer = stripe.Customer.retrieve(customer_obj.stripe_id)
try:
card = customer.sources.create(
source={
"object": "card",
"number": number,
"exp_month": expiry_month,
"exp_year": expiry_year,
"cvc": cvc,
"name": name
}
)
# making it the default card
customer.default_source = card.id
customer.save()
except CardError as ce:
logger.error("Got CardError for customer_id={0}, CardError={1}".format(customer_obj.pk, ce.json_body))
return Response({"success": False, "error": "Failed to add card"})
else:
customer_obj.card_last_4 = card.get('last4')
customer_obj.card_kind = card.get('type', '')
customer_obj.card_fingerprint = card.get('fingerprint')
customer_obj.save()
return Response({"success": True})
This is the method for unit testing:
#mock.patch('stripe.Customer.retrieve')
#mock.patch('stripe.Customer.create')
def test_add_card(self,create_mock,retrieve_mock):
response = {
'default_card': None,
'cards': {
"count": 0,
"data": []
}
}
# save_mock.return_value = response
create_mock.return_value = response
retrieve_mock.return_value = response
self.api_client.client.login(username = self.username, password = self.password)
res = self.api_client.post('/biz/api/auth/card/add')
print res
Now stripe.Customer.retrieve is being mocked properly. But I am not able to mock customer.sources.create. I am really stuck on this.
This is the right way of doing it:
#mock.patch('stripe.Customer.retrieve')
def test_add_card_failure(self, retrieve_mock):
data = {
'name': "shubham",
'cvc': 123,
'number': "4242424242424242",
'expiry': "12/23",
}
e = CardError("Card Error", "", "")
retrieve_mock.return_value.sources.create.return_value = e
self.api_client.client.login(username=self.username, password=self.password)
res = self.api_client.post('/biz/api/auth/card/add', data=data)
self.assertEqual(self.deserialize(res)['success'], False)
Even though the given answer is correct, there is a way more comfortable solution using vcrpy. That is creating a cassette (record) once a given record does not exist yet. When it does, the mocking is done transparently and the record will be replayed. Beautiful.
Having a vanilla pyramid application, using py.test, my test now looks like this:
import vcr
# here we have some FactoryBoy fixtures
from tests.fixtures import PaymentServiceProviderFactory, SSOUserFactory
def test_post_transaction(sqla_session, test_app):
# first we need a PSP and a User existent in the DB
psp = PaymentServiceProviderFactory() # type: PaymentServiceProvider
user = SSOUserFactory()
sqla_session.add(psp, user)
sqla_session.flush()
with vcr.use_cassette('tests/casettes/tests.checkout.services.transaction_test.test_post_transaction.yaml'):
# with that PSP we create a new PSPTransaction ...
res = test_app.post(url='/psps/%s/transaction' % psp.id,
params={
'token': '4711',
'amount': '12.44',
'currency': 'EUR',
})
assert 201 == res.status_code
assert 'id' in res.json_body
IMO, the following method is better than the rest of the answers
import unittest
import stripe
import json
from unittest.mock import patch
from stripe.http_client import RequestsClient # to mock the request session
stripe.api_key = "foo"
stripe.default_http_client = RequestsClient() # assigning the default HTTP client
null = None
false = False
true = True
charge_resp = {
"id": "ch_1FgmT3DotIke6IEFVkwh2N6Y",
"object": "charge",
"amount": 1000,
"amount_captured": 1000,
"amount_refunded": 0,
"billing_details": {
"address": {
"city": "Los Angeles",
"country": "USA",
},
"email": null,
"name": "Jerin",
"phone": null
},
"captured": true,
}
def get_customer_city_from_charge(stripe_charge_id):
# this is our function and we are writing unit-test for this function
charge_response = stripe.Charge.retrieve("foo-bar")
return charge_response.billing_details.address.city
class TestStringMethods(unittest.TestCase):
#patch("stripe.default_http_client._session")
def test_get_customer_city_from_charge(self, mock_session):
mock_response = mock_session.request.return_value
mock_response.content.decode.return_value = json.dumps(charge_resp)
mock_response.status_code = 200
city_name = get_customer_city_from_charge("some_id")
self.assertEqual(city_name, "Los Angeles")
if __name__ == '__main__':
unittest.main()
Advantages of this method
You can generate the corresponding class objects (here, the charge_response variable is a type of Charge--(source code))
You can use the dot (.) operator over the response (as we can do with real stripe SDK)
dot operator support for deep attributes