I am having a problem when I get the value from a hstore field. The situation is something like this:
model:
class Model(models.Model):
hstoreField = HStoreField()
test cases:
def test_case_1(self):
new_model = Model.objects.create(
hstoreField = {
'key1': 'value1',
'key2': 'value2'
}
)
self.assertEqual(new_model.subscription['key1'], 'value1')
self.assertEqual(new_model.subscription['key2'], 'value2')
def test_case_2(self):
Model.objects.create(
hstoreField = {
'key1': 'value1',
'key2': 'value2'
}
)
new_model = Model.objects.get( id = 1 ) // assume it works
self.assertEqual(new_model.hstoreField['key1'], 'value1')
self.assertEqual(new_model.hstoreField['key2'], 'value2')
When I run those test cases, the first one passes, but the second one fails and the error thrown is: TypeError: string indices must be integers. Checking the value of hstoreField is a string and it should be a dict.
The error only happens in tests, in real code both cases work. Can you help me guys?
Related
The issue
I'm trying out great expectations with dagster, as per this guide
My pipeline seems to execute correctly until it reaches this block:
expectation = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='dev.data-pipeline-data-storage.data_pipelines.raw_data.sirene_update',
suite_name='suite.data_pipelines.raw_data.sirene_update',
)
if expectation["success"]:
print("Success")
trying to call expectation["success"] results in a
# TypeError: 'SolidDefinition' object is not subscriptable
When I go inside the code of ge_validation_op_factory, there is a _ge_validation_fn that should yield ExpectationResult, but somehow it gets coverted into a SolidDefinition...
Dagster version = 0.15.9;
Great Expectations version = 0.15.44
Code to reproduce the error
In my code, I am trying to interact with an s3 bucket, so it would be a bit tedious to re-create the code for my example but here it is anyway:
In a gx_postprocessing.py
import json
import boto3
import dagster_ge
from dagster import (
op,
graph,
Field,
String,
OpExecutionContext,
)
from typing import List, Dict
#op(
config_schema={
"bucket": Field(
String,
description="s3 bucket name",
),
"path_in_s3": Field(
String,
description="Prefix representing the path to data",
),
"technical_date": Field(
String,
description="date string to fetch data",
),
"file_name": Field(
String,
description="file name that contains the data",
),
}
)
def read_in_json_datafile_from_s3(context: OpExecutionContext):
bucket = context.op_config["bucket"]
path_in_s3 = context.op_config["path_in_s3"]
technical_date = context.op_config["technical_date"]
file_name = context.op_config["file_name"]
object = f"{path_in_s3}/" f"technical_date={technical_date}/" f"{file_name}"
s3 = boto3.resource("s3")
content_object = s3.Object(bucket, object)
file_content = content_object.get()["Body"].read().decode("utf-8")
json_content = json.loads(file_content)
return json_content
#op
def process_example_dq(data: List[Dict]):
return len(data)
#op
def postprocess_example_dq(numrows, expectation):
if expectation["success"]:
return numrows
else:
raise ValueError
#op
def validate_example_dq(context: OpExecutionContext):
expectation = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='my_bucket.data_pipelines.raw_data.example_update',
suite_name='suite.data_pipelines.raw_data.example_update',
)
return expectation
#graph(
config={
"read_in_json_datafile_from_s3": {
"config": {
"bucket": "my_bucket",
"path_in_s3": "my_path",
"technical_date": "2023-01-24",
"file_name": "myfile_20230124.json",
}
},
},
)
def example_update_evaluation():
output_dict = read_in_json_datafile_from_s3()
nb_items = process_example_dq(data=output_dict)
expectation = validate_example_dq()
postprocess_example_dq(
numrows=nb_items,
expectation=expectation,
)
Do not forget to add great_expectations_poc_pipeline to your __init__.py where the pipelines=[..] are listed.
In this example, dagster_ge.ge_validation_op_factory(...) is returning an OpDefinition, which is the same type of thing as (for example) process_example_dq, and should be composed in the graph definition the same way, rather than invoked within another op.
So instead, you'd want to have something like:
validate_example_dq = dagster_ge.ge_validation_op_factory(
name='ge_validation_op',
datasource_name='my_bucket.data_pipelines.raw_data.example_update',
suite_name='suite.data_pipelines.raw_data.example_update',
)
Then use that op inside your graph definition the same way you currently are (i.e. expectation = validate_example_dq())
I am trying to return most voted answer for each question.also i want to send also extra infomation of that answer like vote and id.
Printing one value is easy but for more than one i have to return dictionary.So how can i return dictionary and print all values in template.
from django import template
register = template.Library()
#register.simple_tag
def getmostvotedanswer(answers):
answer = answers.order_by('-vote')[0]
answer_info = {
'answer':answer.answer,
'vote':answer.vote,
'id':answer.id
}
return answer_info
index.html
<p class="small text-muted ">{% getmostvotedanswer question.answer_set.all %}</p>
Output
{'answer': 'THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER THIS IS ANSWER', 'vote': 7, 'id': 1}
I can call template_tag 3 times for three values.
But I don't want to call templatetag again and again i think it will affect performance.
view.py
def index(request):
questions = Question.objects.all()
context = {
'questions':questions
}
return render(request,'index.html',context=context)
Edit -> Add view.py
The easiest, debugable, best performance and test friendly way to achieve it is to cook data on view instead of write custom template tag. Windows functions are needed to get the first answer of each question:
from django.db.models import F, Window
from django.db.models.functions.window import FirstValue
def index(request):
#q_and_a_ids = [ (id question, id most voted answer), (... ]
q_and_a_ids = (
Question
.objects
.annotate(
most_voted_id=Window(
expression=FirstValue('answer__id'),
partition_by=['id'],
order_by=F('answer__vote').desc()
)
)
.distinct()
.values_list( 'id', 'most_voted_id')
)
answers_ids = set( [ a_id for (_,a_id) in q_and_a_ids] )
questions_dict = Question.objects.in_bulk()
answers_dict = Answers.objects.filter(pk__in=answers_ids).in_bulk()
#q_and_a = [ { 'q':question, 'a':most voted answer}, { ... ]
q_and_a = [ {'q': questions_dict[q_id],
'a': answers_dict.get(a_id) }
for (q_id,a_id) in q_and_a_ids ]
context = {
'questions_and_answers': q_and_a
}
return render(request,'index.html',context=context)
In tastypie, I want set json result name.
I have a class that I use for it but I can set name in.
enter cclass ContentResource(ModelResource):
class Meta:
results = ListField(attribute='results')
queryset = Content.objects.all()
resource_name = 'content'
max_limit = None
#filtering = {"title": "contains"}
def alter_list_data_to_serialize(self, request, data_dict):
if isinstance(data_dict, dict):
if 'meta' in data_dict:
# Get rid of the "meta".
del(data_dict['meta'])
# Rename the objects.
data_dict['Mobile'] = data_dict['objects']
del(data_dict['objects'])
return data_dict
ode here it returns this
{"Mobile":
[
{
"added": "2015-07-23T11:30:20.911835",
"content_cast": "",
"content_company": "HamrahCinema",
"content_description": "so nice",
"content_director": "",
"content_duration": "2:20",
"content_filelanguage": null,
}
]
}
when I use /content/api/content every thing is ok, but when I use /content/api/content/1,"mobile" is removed.
as educated guess, I would suggest using alter_detail_data_to_serialize
I have a model in django admin as follows
ChoiceA= (
("on-false","on-false"),
("on-true","on-true"),
)
ChoiceB = (
("always","always"),
("never","never"),
)
id = models.CharField(verbose_name="Field",max_length=32)
type = models.CharField(verbose_name="Expression",max_length=32)
action = models.CharField(max_length=32, choices=x)
Now based on the type entered by the user ie if user enters type = "a" then action's choices should be set to ChoiceA and if user enters type ="b" then action's choices should be set to ChoiceB. How can I achieve this in Django Admin?
Edit:
action_change.js
jQuery(document).ready(function(){
$("#id_type").change( function(event) {
$.ajax({
"type" : "POST",
"url" : "/action_choices/",
"dataType" : "json",
"cache" : false,
"error" : alert("hello"),
"success" : function(json) {
$('#id_action >option').remove();
for(var j = 0; j < json.length; j++){
$('#id_action').append($('<option></option>').val(json[j][0]).html(json[j][1]));
}
}
});
});
});
You can achieve it using Ajax and jQuery:
models.py:
type = models.CharField(verbose_name="Expression",max_length=32)
action = models.CharField(max_length=32, choices = (('', ''), ))
admin.py:
class MyModelAdmin(admin.ModelAdmin):
list_display = ('type', )
class Media:
js = ['/static/js/action_change.js']
admin.site.register(MyModel, MyModelAdmin)
urls.py:
url(r'^action_choices/', 'myproject.myapp.views.action_choices'),
views.py:
def action_choices(request):
action_list = []
ChoiceA = ("on-false", "on-true")
ChoiceB = ("always", "never")
action_type = request.GET.get('action_type')
if str(action_type).lower() == 'a':
choices = ChoiceA
elif str(action_type).lower() == 'b':
choices = ChoiceB
else:
choices = ()
[action_list.append((each,each)) for each in choices]
json = simplejson.dumps(action_list)
return HttpResponse(json, mimetype='application/javascript')
Create the file action_change.js with following content in your static folder and define correct path in class Media of ModelAdmin.
action_change.js
(function($){
$(function(){
$(document).ready(function() {
$('#id_type').bind('keyup', type_change);
$('#id_action >option').show();
});
});
})(django.jQuery);
// based on the type, action will be loaded
var $ = django.jQuery.noConflict();
function type_change()
{
var action_type = $('#id_type').val();
$.ajax({
"type" : "GET",
"url" : "/action_choices/?action_type="+action_type,
"dataType" : "json",
"cache" : false,
"success" : function(json) {
$('#id_action >option').remove();
for(var j = 0; j < json.length; j++){
$('#id_action').append($('<option></option>').val(json[j][0]).html(json[j][1]));
}
}
})(jQuery);
}
This should work fine for the scenario you asked. And I'm giving my suggestion below:
models.py
type = models.CharField(verbose_name="Expression",max_length=32, choices = (('a', 'a'), ('b', 'b'), ))
action = models.CharField(max_length=32, choices = (('', ''), ))
action_change.js (line 5)
$('#id_type').bind('change', type_change);
You would have to initialize the action field with all possible choices, or Django will complain that a choice that didn't previously exist isn't a valid choice.
My recommendation would be to initialize the field with all of the possible choices, and use JavaScript to toggle the visibility of the choices, depending on the value of type. There are a few plugins around that will handle dynamic fields in Django admin, but most that I've seen deal with ForeignKey or ManyToMany fields that need to do lookups.
You're probably best off just adding some JavaScript to your admin form via the Media meta class and handling it yourself.
I'm trying to use a string as a parameter in a function. Is something like this possible?
from django.contrib.auth.models import User
param_var = 'contains'
User.objects.filter(first_name__{param_var}='John')
# ideally, this should equate to:
User.objects.filter(first_name__contains='John')
param_var = 'contains'
User.objects.filter(**{'first_name__%s'%param_var: 'John'})
Though I really wonder about the wisdom of doing that... I think you may be re-inventing sql injection...
I suspect there's a better way. You should probably explain more of what you are trying to accomplish.
Solved:
# Something like this...
qs = User.objects.all()
parameters = {
'name': {
'type': 'get',
'filter': [
'first_name__icontains',
'last_name__icontains',
# ...
],
},
# ...
}
for parameter, data in parameters.items():
if data['type'] == 'get':
value = request.GET.get(parameter, None)
elif data['type'] == 'getlist':
value = request.GET.getlist(parameter)
if value:
query = Q()
for f in data['filter']:
query = query | Q(**{ f : value })
qs = qs.filter(query)