no search results Django 2.1 Solr 6.5 - django

I implemented solr 6.5 for django 2.1 with haystack 2.5 and i succesfully built index but theres still no search results
q*:* returns
{
"responseHeader":{
"status":0,
"QTime":0,
"params":{
"q":"*:*",
"indent":"on",
"wt":"json",
"_":"1543140724323"}},
"response":{"numFound":5,"start":0,"docs":[
{
"id":"blog.post.1",
"django_ct":["blog.post"],
"publish":["2019-05-26T04:46:00+00:00Z"],
"text":["Chwalmy wodza\nsanacja, wódz, historia, sumienie\nBo jeden jest wódz\n\n"],
"django_id":[1],
"_version_":1618099571920994304},
{
"id":"blog.post.3",
"django_ct":["blog.post"],
"publish":["2018-11-15T20:59:39+00:00Z"],
"text":["Trwoga Krzyk Gardłowy\nwódz\nsome\n\n"],
"django_id":[3],
"_version_":1618099571923091456},
{
"id":"blog.post.4",
"django_ct":["blog.post"],
"publish":["2018-11-15T21:24:01+00:00Z"],
"text":["Jeszcze jeden post\nsanacja, wódz\nChwalmy wodza\n\n"],
"django_id":[4],
"_version_":1618099571924140032},
{
"id":"blog.post.5",
"django_ct":["blog.post"],
"publish":["2018-11-21T14:41:25+00:00Z"],
"text":["Yep\nsanacja, wódz\nffff\n\n"],
"django_id":[5],
"_version_":1618099571925188608},
{
"id":"blog.post.7",
"django_ct":["blog.post"],
"publish":["2018-11-21T21:30:30+00:00Z"],
"text":["Markdown\nhistoria, sanacja, wódz\n*wyróżnienie* **pogrubienie**\nlistę:\n\n\n\n\nSometimes you want bullet points:\n\n* Start a line with a star\n* Profit!\n* Start a line with a star\n* Else\n\nTutaj jest [sabaton](https://www.youtube.com/watch?v=_HLNMfCBUb0)\n\n"],
"django_id":[7],
"_version_":1618099571925188609}]
So core is populated
My view:
def post_search(request):
form = SearchForm()
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
cd = form.cleaned_data
results = SearchQuerySet().models(Post)\
.filter(content=cd['query']).load_all()
# Obliczenie całkowitej liczby wyników.
total_results = results.count()
return render(request,
'blog/post/search.html',
{'form': form,
'cd': cd,
'results': results,
'total_results': total_results})
else:
return render(request,
'blog/post/search.html',
{'form': form})
And my form
class SearchForm(forms.Form):
query = forms.CharField()
I tried procedure proposed by haystack itself
>>> from haystack.query import SearchQuerySet
>>> sqs = SearchQuerySet().all()
>>> sqs.count()
But I only recived
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/query.py", line 522, in count
return len(self)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/query.py", line 86, in __len__
self._result_count = self.query.get_count()
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/__init__.py", line 619, in get_count
self.run()
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 802, in run
results = self.backend.search(final_query, **search_kwargs)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/__init__.py", line 33, in wrapper
return func(obj, query_string, *args, **kwargs)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 149, in search
distance_point=kwargs.get('distance_point'))
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 425, in _process_results
app_label, model_name = raw_result[DJANGO_CT].split('.')
AttributeError: 'list' object has no attribute 'split'
So here's the problem, posts are indexed but unable to search and I really run out of ideas at this point

Related

runtime error. there is no current event loop in thread 'Thread-1'

i followed this tutorial https://python.gotrained.com/scraping-telegram-group-members-python-telethon/, and the code ran successfully on a single python script. so i decided to implement the same code into django views.py. form.py, html templates and urls.py are properly connected. whenever i runserver and enter details it raises runtime error saying "There is no current event loop in thread 'Thread-1'."
views.py
import asyncio
from django.shortcuts import render,redirect
from telethon.sync import TelegramClient
from .form import DetailForm,CodeForm
from .models import Detail,Code
# Create your views here.
def home(request):
form = DetailForm
context = {'form':form}
if request.method=='POST':
form = DetailForm(request.POST)
if form.is_valid():
form.save()
details = Detail.objects.all().last()
api_id = int(details.api_id)
api_hash = details.api_hash
phone_number = details.phone_number
loop = asyncio.new_event_loop()
client = TelegramClient(phone_number, api_id, api_hash, loop=loop)
client.connect()
if not client.is_user_authorized():
client.send_code_request(phone_number)
# global client, phone_number
return redirect('code')
return render(request, 'teleapp/home.html', context)
def codeview(request):
code=CodeForm
# codemodel = Code.objects.all().last()
if request.method == 'POST':
codes = CodeForm(request.POST)
if codes.is_valid():
codes.save()
#what next...
return redirect('scrapy')
context = {'code':code}
return render(request, 'teleapp/code.html', context)
def scrappy(request):
details = Detail.objects.all().last()
context = {'details':details}
return render(request, 'teleapp/scrappy.html', context)```
**error raised**
Internal Server Error: /
Traceback (most recent call last):
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\User\PycharmProjects\telegram\scrapy\teleapp\views.py", line
24, in home
client.connect()
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\telethon
\sync.py", line 35, in syncified
loop = asyncio.get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\ev
ents.py", line 694, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\events.py", line 602, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-2'.

AttributeError: 'tuple' object has no attribute 'get' in views.py

i want to create a registration form but i m getting an attribute error while running the application
from django.shortcuts import render
from basic_app.forms import UserForm,UserProfileInfoForm
def index(request):
return render(request,'basic_app/index.html')
def register(request):
registered=False
if request.method=="POST":
user_form=UserForm(data=request.POST)
profile_form=UserProfileInfoForm(data=request.POST)
if user_form.is_valid() and profile_from.is_valid():
user=user_form.save()
user.setpassword(user.password)
user.save()
profile=profile_form.save(commit=False)
profile.user=user
if 'profile_pic' in request.FILES:
profile.profile_pic=request.FILES['profile_pic']
profile.save()
registered=True
else:
print(user_form.errors,profile_form.errors)
else:
user_form=UserForm()
profile_form=UserProfileInfoForm()
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
output
Internal Server Error: /basic_app/register/
Traceback (most recent call last):
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\utils\deprecation.py", line 93, in call
response = self.process_response(request, response)
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\middleware\clickjacking.py", line 26, in process_response
if response.get('X-Frame-Options') is not None:
AttributeError: 'tuple' object has no attribute 'get'
[24/Dec/2018 15:34:51] "GET /basic_app/register/ HTTP/1.1" 500 61448
That is because of this line:
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
You've actually created a tuple here. Notice the the parenthesis around the three things that you're returning?
This is how you render a template in django:
from django.shortcuts import render
render(request, 'polls/index.html', context)
So in your case this will work:
render(request,'basic_app/registration.html', {
'user_form':user_form,
'profile_form':profile_form,
'registered':registered
})
For more information on render check out its docs
Change this
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
to
return render(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})

scrapy TypeError: object() takes no parameters

I am new to Scrapy and trying to crawl a couple of links as a test using Scrapy. Whenever I run scrapy crawl tier1, I get "TypeError: object() takes no parameters" as the following:
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
2017-08-23 17:25:02 [tier1-parse-logger] INFO: Entered the parse function to parse and index: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<date>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<author>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [scrapy.core.scraper] ERROR: Spider error processing <GET http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166> (referer: None)
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
And, my spider file (tier1_crawler.py):
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
sys.path.append(os.path.abspath('..'))
import logging
import scrapy
from scrapy.loader import ItemLoader
from adv_crawler.items import AdvCrawlerItem
from datetime import datetime, date, time
t1_parse_logger = logging.getLogger("tier1-parse-logger")
t1_parse_logger.LOG_FILE = "Tier1-log.txt"
content_type_dic = {
'news': 'news',
}
class Tier1Crawler(scrapy.Spider):
name = "tier1"
def start_requests(self):
urls = ['http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535982',
'http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
t1_parse_logger.info("Entered the parse function to parse and index: %s" % response.url) # Log at the beginning of the parse function
item_loader = ItemLoader(item=AdvCrawlerItem(), response=response)
if 'mk.co.kr' in response.url:
mk_loader = item_loader.nested_xpath('//div[#id="top_header"]/div[#class="news_title"]/div[#class="news_title_text"]')
try:
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime"]')
except AttributeError: # if the date is not in "lasttime" li tag
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime1"]')
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<date>> from a mk article: %s" % response.url)
try:
mk_loader.add_xpath('author', 'div[#class="news_title_author"]/ul/li[#class="author"]')
except AttributeError: # in case there is no author (some mk articles have no author)
item_loader.add_value('author', "None") # ir error, replace with the line below
# item['author'] = "None" # if the above gives any error, replace the above with this line
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<author>> from a mk article: %s" % response.url)
item_loader.add_xpath('content', '//div[#id="Content"]/div[#class="left_content"]/div[#id="article_body"]/div[#class="art_txt"]') # Content of the article (entire contents)
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
item_loader.add_value('content_type', content_type_dic['news'])
item_loader.add_value('timestamp', str(datetime.now())) # timestamp of when the document is being indexed
item_loader.add_value('url', response.url) # url of the article
t1_parse_logger.info("Parsed and indexed: %s" % response.url)
return item_loader.load_item()
And, my items.py file:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_date(value):
if isinstance(value, unicode):
(year, month, day) = str(value.split(" ")[-2]).split(".")
return year+"-"+month+"-"+day
def filter_utf(value):
if isinstance(value, unicode):
return value.encode('utf-8')
class AdvCrawlerItem(scrapy.Item):
author = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # Name of the publisher/author
content = scrapy.Field(input_processor=MapCompose(remove_tags, Join, filter_utf),) # Content of the article (entire contents)
content_type = scrapy.Field()
date = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_date),)
timestamp = scrapy.Field() # timestamp of when the document is being indexed
title = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # title of the article
url = scrapy.Field() # url of the article
And, pipelines.py file:
import json
from scrapy import signals
from scrapy.exporters import JsonLinesItemExporter
class AdvCrawlerJsonExportPipeline(object):
def open_spider(self, spider):
self.file = open('crawled-articles1.txt', 'w')
def close_spider(self, spider):
self.file.close()
def process_item(self, item, spider):
line = json.dummps(dict(item)) + "\n"
self.file.write(line)
return item
I am aware that "TypeError: object() takes no parameters" error is usually thrown when __init__ method of a class is not defined at all or not defined to take in parameter(s).
However, in the case above, how can i fix the error? Am I doing something wrong using the item loader or nested item loader??
When using scrapy processors you should use the classes to create objects that do the processing:
# wrong
field = Field(output_processor=MapCompose(TakeFirst))
# right
field = Field(output_processor=MapCompose(TakeFirst()))
^^

Flask-Login: KeyError: 'security' when trying to call login_user

I have the following code that logs in a user on my Flask application:
#app.route('/login', methods = ['GET', 'POST'])
def login():
if request.method == 'GET':
return render_template('login.html')
db = get_db()
member = member_from_phone(request.form['phone_number'])
member_obj = Member(member)
if member is None:
g.result = json.dumps({'head' : 200, 'body' : "invalid phone" })
return 'no member...'
elif request.form['password'] == member['password']:
login_user(member_obj)
return 'login successful'
else:
return 'login failed'
return home_page()
However it throws a KeyError: 'security' when I try log in:
File "/Library/Python/2.7/site-packages/flask_security/utils.py", line 64, in login_user
if _security.trackable:
File "/Library/Python/2.7/site-packages/werkzeug/local.py", line 338, in __getattr__
return getattr(self._get_current_object(), name)
File "/Library/Python/2.7/site-packages/werkzeug/local.py", line 297, in _get_current_object
return self.__local()
File "/Library/Python/2.7/site-packages/flask_security/utils.py", line 35, in <lambda>
_security = LocalProxy(lambda: current_app.extensions['security'])
KeyError: 'security' when trying to call login_user
I have no idea what's going on, I think I might need to fix my flask configuration but I don't know how.

FormWizard and saving data

im trying to use django formWizard, all is fine, but im getting an error in the last step after submit
Traceback (most recent call last):
File "/home/vacantes/webapps/django/lib/python2.6/django/core/handlers/base.py", line 100, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 21, in _wrapper
return decorator(bound_func)(*args, **kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 76, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 17, in bound_func
return func(self, *args2, **kwargs2)
File "/home/vacantes/webapps/django/lib/python2.6/django/contrib/formtools/wizard.py", line 101, in __call__
return self.done(request, final_form_list)
File "/home/vacantes/webapps/django/hay_vacantes/curriculums/forms.py", line 56, in done
website = data['website']
TypeError: __init__() got an unexpected keyword argument 'website'
i dont know where is the problem, in my Model i have a website field, but ... why the error?
model.py
class Generales(models.Model):
usuario = models.OneToOneField(User,unique=True)
.....
estado_civil = models.SmallIntegerField(max_length=1,choices=CIVIL)
website = models.URLField(verbose_name='Website', verify_exists=False,null=True,blank=True)
and in my forms:
forms.py
#others forms
.....
class ContactWizard(FormWizard):
def done(self, request, form_list):
data = {}
for form in form_list:
data.update(form.cleaned_data)
generales = Generales(
usuario = request.user.id,
documento_identidad = data['documento_identidad'],
tipo = data['tipo'],
tel_fijo = data['tel_fijo'],
celular = data['celular'],
foto = data['foto'],
sexo = data['sexo'],
direccion = data['direccion'],
codigo_postal = data['codigo_postal'],
pais = data['pais'],
ciudad = data['ciudad'],
fecha_nacimiento = data['fecha_nacimiento'],
estado_civil = data['estado_civil'],
website = data['website']
)
generales.save()
# others forms
.....
return HttpResponseRedirect('/panel/completo/')
EDIT
urls.py
from vacantes.curriculums.forms import Generales,Areas,Experiencia,Referencias,ContactWizard
urlpatterns = patterns('',
url(r'^completar/$', ContactWizard([Generales, Areas,Experiencia,Referencias])),
)
i dont know if im saving the data like formwizard need, but im trying.
any idea about the error?
thanks
As i have already used FormWizard in my app, the keys of submitted data are not 'website' or 'estado_civil',... but in the form "< form_number>-< field_name>". For example: '0-website' if the field 'website' is in the first form, '1-website' if it is in the second form and so on.
You can print out the whole submitted data dictionary to check that.