I implemented solr 6.5 for django 2.1 with haystack 2.5 and i succesfully built index but theres still no search results
q*:* returns
{
"responseHeader":{
"status":0,
"QTime":0,
"params":{
"q":"*:*",
"indent":"on",
"wt":"json",
"_":"1543140724323"}},
"response":{"numFound":5,"start":0,"docs":[
{
"id":"blog.post.1",
"django_ct":["blog.post"],
"publish":["2019-05-26T04:46:00+00:00Z"],
"text":["Chwalmy wodza\nsanacja, wódz, historia, sumienie\nBo jeden jest wódz\n\n"],
"django_id":[1],
"_version_":1618099571920994304},
{
"id":"blog.post.3",
"django_ct":["blog.post"],
"publish":["2018-11-15T20:59:39+00:00Z"],
"text":["Trwoga Krzyk Gardłowy\nwódz\nsome\n\n"],
"django_id":[3],
"_version_":1618099571923091456},
{
"id":"blog.post.4",
"django_ct":["blog.post"],
"publish":["2018-11-15T21:24:01+00:00Z"],
"text":["Jeszcze jeden post\nsanacja, wódz\nChwalmy wodza\n\n"],
"django_id":[4],
"_version_":1618099571924140032},
{
"id":"blog.post.5",
"django_ct":["blog.post"],
"publish":["2018-11-21T14:41:25+00:00Z"],
"text":["Yep\nsanacja, wódz\nffff\n\n"],
"django_id":[5],
"_version_":1618099571925188608},
{
"id":"blog.post.7",
"django_ct":["blog.post"],
"publish":["2018-11-21T21:30:30+00:00Z"],
"text":["Markdown\nhistoria, sanacja, wódz\n*wyróżnienie* **pogrubienie**\nlistę:\n\n\n\n\nSometimes you want bullet points:\n\n* Start a line with a star\n* Profit!\n* Start a line with a star\n* Else\n\nTutaj jest [sabaton](https://www.youtube.com/watch?v=_HLNMfCBUb0)\n\n"],
"django_id":[7],
"_version_":1618099571925188609}]
So core is populated
My view:
def post_search(request):
form = SearchForm()
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
cd = form.cleaned_data
results = SearchQuerySet().models(Post)\
.filter(content=cd['query']).load_all()
# Obliczenie całkowitej liczby wyników.
total_results = results.count()
return render(request,
'blog/post/search.html',
{'form': form,
'cd': cd,
'results': results,
'total_results': total_results})
else:
return render(request,
'blog/post/search.html',
{'form': form})
And my form
class SearchForm(forms.Form):
query = forms.CharField()
I tried procedure proposed by haystack itself
>>> from haystack.query import SearchQuerySet
>>> sqs = SearchQuerySet().all()
>>> sqs.count()
But I only recived
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/query.py", line 522, in count
return len(self)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/query.py", line 86, in __len__
self._result_count = self.query.get_count()
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/__init__.py", line 619, in get_count
self.run()
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 802, in run
results = self.backend.search(final_query, **search_kwargs)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/__init__.py", line 33, in wrapper
return func(obj, query_string, *args, **kwargs)
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 149, in search
distance_point=kwargs.get('distance_point'))
File "/home/hesmeron/first_env/lib/python3.5/site-packages/haystack/backends/solr_backend.py", line 425, in _process_results
app_label, model_name = raw_result[DJANGO_CT].split('.')
AttributeError: 'list' object has no attribute 'split'
So here's the problem, posts are indexed but unable to search and I really run out of ideas at this point
Related
i followed this tutorial https://python.gotrained.com/scraping-telegram-group-members-python-telethon/, and the code ran successfully on a single python script. so i decided to implement the same code into django views.py. form.py, html templates and urls.py are properly connected. whenever i runserver and enter details it raises runtime error saying "There is no current event loop in thread 'Thread-1'."
views.py
import asyncio
from django.shortcuts import render,redirect
from telethon.sync import TelegramClient
from .form import DetailForm,CodeForm
from .models import Detail,Code
# Create your views here.
def home(request):
form = DetailForm
context = {'form':form}
if request.method=='POST':
form = DetailForm(request.POST)
if form.is_valid():
form.save()
details = Detail.objects.all().last()
api_id = int(details.api_id)
api_hash = details.api_hash
phone_number = details.phone_number
loop = asyncio.new_event_loop()
client = TelegramClient(phone_number, api_id, api_hash, loop=loop)
client.connect()
if not client.is_user_authorized():
client.send_code_request(phone_number)
# global client, phone_number
return redirect('code')
return render(request, 'teleapp/home.html', context)
def codeview(request):
code=CodeForm
# codemodel = Code.objects.all().last()
if request.method == 'POST':
codes = CodeForm(request.POST)
if codes.is_valid():
codes.save()
#what next...
return redirect('scrapy')
context = {'code':code}
return render(request, 'teleapp/code.html', context)
def scrappy(request):
details = Detail.objects.all().last()
context = {'details':details}
return render(request, 'teleapp/scrappy.html', context)```
**error raised**
Internal Server Error: /
Traceback (most recent call last):
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\django\c
ore\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\User\PycharmProjects\telegram\scrapy\teleapp\views.py", line
24, in home
client.connect()
File "C:\Users\User\PycharmProjects\telegram\venv\lib\site-packages\telethon
\sync.py", line 35, in syncified
loop = asyncio.get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\ev
ents.py", line 694, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\asyncio\events.py", line 602, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-2'.
i want to create a registration form but i m getting an attribute error while running the application
from django.shortcuts import render
from basic_app.forms import UserForm,UserProfileInfoForm
def index(request):
return render(request,'basic_app/index.html')
def register(request):
registered=False
if request.method=="POST":
user_form=UserForm(data=request.POST)
profile_form=UserProfileInfoForm(data=request.POST)
if user_form.is_valid() and profile_from.is_valid():
user=user_form.save()
user.setpassword(user.password)
user.save()
profile=profile_form.save(commit=False)
profile.user=user
if 'profile_pic' in request.FILES:
profile.profile_pic=request.FILES['profile_pic']
profile.save()
registered=True
else:
print(user_form.errors,profile_form.errors)
else:
user_form=UserForm()
profile_form=UserProfileInfoForm()
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
output
Internal Server Error: /basic_app/register/
Traceback (most recent call last):
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\utils\deprecation.py", line 93, in call
response = self.process_response(request, response)
File "C:\Users\Shoaib Khan\AppData\Local\conda\conda\envs\myenv\lib\site-packages\django\middleware\clickjacking.py", line 26, in process_response
if response.get('X-Frame-Options') is not None:
AttributeError: 'tuple' object has no attribute 'get'
[24/Dec/2018 15:34:51] "GET /basic_app/register/ HTTP/1.1" 500 61448
That is because of this line:
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
You've actually created a tuple here. Notice the the parenthesis around the three things that you're returning?
This is how you render a template in django:
from django.shortcuts import render
render(request, 'polls/index.html', context)
So in your case this will work:
render(request,'basic_app/registration.html', {
'user_form':user_form,
'profile_form':profile_form,
'registered':registered
})
For more information on render check out its docs
Change this
return(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
to
return render(request,'basic_app/registration.html',
{'user_form':user_form,
'profile_form':profile_form,
'registered':registered})
I am new to Scrapy and trying to crawl a couple of links as a test using Scrapy. Whenever I run scrapy crawl tier1, I get "TypeError: object() takes no parameters" as the following:
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
2017-08-23 17:25:02 [tier1-parse-logger] INFO: Entered the parse function to parse and index: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<date>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [tier1-parse-logger] ERROR: Error (object() takes no parameters) when trying to parse <<author>> from a mk article: http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166
2017-08-23 17:25:02 [scrapy.core.scraper] ERROR: Spider error processing <GET http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166> (referer: None)
Traceback (most recent call last):
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/adv_crawler/adv_crawler/spiders/tier1_crawler.py", line 93, in parse
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 167, in add_xpath
self.add_value(field_name, values, *processors, **kw)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 77, in add_value
self._add_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 91, in _add_value
processed_value = self._process_input_value(field_name, value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/__init__.py", line 150, in _process_input_value
return proc(value)
File "/Users/btaek/TaeksProgramming/adv/crawler/lib/python2.7/site-packages/scrapy/loader/processors.py", line 28, in __call__
next_values += arg_to_iter(func(v))
TypeError: object() takes no parameters
And, my spider file (tier1_crawler.py):
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
sys.path.append(os.path.abspath('..'))
import logging
import scrapy
from scrapy.loader import ItemLoader
from adv_crawler.items import AdvCrawlerItem
from datetime import datetime, date, time
t1_parse_logger = logging.getLogger("tier1-parse-logger")
t1_parse_logger.LOG_FILE = "Tier1-log.txt"
content_type_dic = {
'news': 'news',
}
class Tier1Crawler(scrapy.Spider):
name = "tier1"
def start_requests(self):
urls = ['http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535982',
'http://news.mk.co.kr/newsRead.php?sc=30000001&year=2017&no=535166',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
t1_parse_logger.info("Entered the parse function to parse and index: %s" % response.url) # Log at the beginning of the parse function
item_loader = ItemLoader(item=AdvCrawlerItem(), response=response)
if 'mk.co.kr' in response.url:
mk_loader = item_loader.nested_xpath('//div[#id="top_header"]/div[#class="news_title"]/div[#class="news_title_text"]')
try:
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime"]')
except AttributeError: # if the date is not in "lasttime" li tag
mk_loader.add_xpath('date', 'div[#class="news_title_author"]/ul/li[#class="lasttime1"]')
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<date>> from a mk article: %s" % response.url)
try:
mk_loader.add_xpath('author', 'div[#class="news_title_author"]/ul/li[#class="author"]')
except AttributeError: # in case there is no author (some mk articles have no author)
item_loader.add_value('author', "None") # ir error, replace with the line below
# item['author'] = "None" # if the above gives any error, replace the above with this line
except Exception as e: # in case the error is not AttributeError
t1_parse_logger.error("Error "+"("+str(e)+")"+" when trying to parse <<author>> from a mk article: %s" % response.url)
item_loader.add_xpath('content', '//div[#id="Content"]/div[#class="left_content"]/div[#id="article_body"]/div[#class="art_txt"]') # Content of the article (entire contents)
mk_loader.add_xpath('title', 'h1[#class="top_title"]') # Title of the article
item_loader.add_value('content_type', content_type_dic['news'])
item_loader.add_value('timestamp', str(datetime.now())) # timestamp of when the document is being indexed
item_loader.add_value('url', response.url) # url of the article
t1_parse_logger.info("Parsed and indexed: %s" % response.url)
return item_loader.load_item()
And, my items.py file:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_date(value):
if isinstance(value, unicode):
(year, month, day) = str(value.split(" ")[-2]).split(".")
return year+"-"+month+"-"+day
def filter_utf(value):
if isinstance(value, unicode):
return value.encode('utf-8')
class AdvCrawlerItem(scrapy.Item):
author = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # Name of the publisher/author
content = scrapy.Field(input_processor=MapCompose(remove_tags, Join, filter_utf),) # Content of the article (entire contents)
content_type = scrapy.Field()
date = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_date),)
timestamp = scrapy.Field() # timestamp of when the document is being indexed
title = scrapy.Field(input_processor=MapCompose(remove_tags, TakeFirst, filter_utf),) # title of the article
url = scrapy.Field() # url of the article
And, pipelines.py file:
import json
from scrapy import signals
from scrapy.exporters import JsonLinesItemExporter
class AdvCrawlerJsonExportPipeline(object):
def open_spider(self, spider):
self.file = open('crawled-articles1.txt', 'w')
def close_spider(self, spider):
self.file.close()
def process_item(self, item, spider):
line = json.dummps(dict(item)) + "\n"
self.file.write(line)
return item
I am aware that "TypeError: object() takes no parameters" error is usually thrown when __init__ method of a class is not defined at all or not defined to take in parameter(s).
However, in the case above, how can i fix the error? Am I doing something wrong using the item loader or nested item loader??
When using scrapy processors you should use the classes to create objects that do the processing:
# wrong
field = Field(output_processor=MapCompose(TakeFirst))
# right
field = Field(output_processor=MapCompose(TakeFirst()))
^^
I have the following code that logs in a user on my Flask application:
#app.route('/login', methods = ['GET', 'POST'])
def login():
if request.method == 'GET':
return render_template('login.html')
db = get_db()
member = member_from_phone(request.form['phone_number'])
member_obj = Member(member)
if member is None:
g.result = json.dumps({'head' : 200, 'body' : "invalid phone" })
return 'no member...'
elif request.form['password'] == member['password']:
login_user(member_obj)
return 'login successful'
else:
return 'login failed'
return home_page()
However it throws a KeyError: 'security' when I try log in:
File "/Library/Python/2.7/site-packages/flask_security/utils.py", line 64, in login_user
if _security.trackable:
File "/Library/Python/2.7/site-packages/werkzeug/local.py", line 338, in __getattr__
return getattr(self._get_current_object(), name)
File "/Library/Python/2.7/site-packages/werkzeug/local.py", line 297, in _get_current_object
return self.__local()
File "/Library/Python/2.7/site-packages/flask_security/utils.py", line 35, in <lambda>
_security = LocalProxy(lambda: current_app.extensions['security'])
KeyError: 'security' when trying to call login_user
I have no idea what's going on, I think I might need to fix my flask configuration but I don't know how.
im trying to use django formWizard, all is fine, but im getting an error in the last step after submit
Traceback (most recent call last):
File "/home/vacantes/webapps/django/lib/python2.6/django/core/handlers/base.py", line 100, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 21, in _wrapper
return decorator(bound_func)(*args, **kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 76, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "/home/vacantes/webapps/django/lib/python2.6/django/utils/decorators.py", line 17, in bound_func
return func(self, *args2, **kwargs2)
File "/home/vacantes/webapps/django/lib/python2.6/django/contrib/formtools/wizard.py", line 101, in __call__
return self.done(request, final_form_list)
File "/home/vacantes/webapps/django/hay_vacantes/curriculums/forms.py", line 56, in done
website = data['website']
TypeError: __init__() got an unexpected keyword argument 'website'
i dont know where is the problem, in my Model i have a website field, but ... why the error?
model.py
class Generales(models.Model):
usuario = models.OneToOneField(User,unique=True)
.....
estado_civil = models.SmallIntegerField(max_length=1,choices=CIVIL)
website = models.URLField(verbose_name='Website', verify_exists=False,null=True,blank=True)
and in my forms:
forms.py
#others forms
.....
class ContactWizard(FormWizard):
def done(self, request, form_list):
data = {}
for form in form_list:
data.update(form.cleaned_data)
generales = Generales(
usuario = request.user.id,
documento_identidad = data['documento_identidad'],
tipo = data['tipo'],
tel_fijo = data['tel_fijo'],
celular = data['celular'],
foto = data['foto'],
sexo = data['sexo'],
direccion = data['direccion'],
codigo_postal = data['codigo_postal'],
pais = data['pais'],
ciudad = data['ciudad'],
fecha_nacimiento = data['fecha_nacimiento'],
estado_civil = data['estado_civil'],
website = data['website']
)
generales.save()
# others forms
.....
return HttpResponseRedirect('/panel/completo/')
EDIT
urls.py
from vacantes.curriculums.forms import Generales,Areas,Experiencia,Referencias,ContactWizard
urlpatterns = patterns('',
url(r'^completar/$', ContactWizard([Generales, Areas,Experiencia,Referencias])),
)
i dont know if im saving the data like formwizard need, but im trying.
any idea about the error?
thanks
As i have already used FormWizard in my app, the keys of submitted data are not 'website' or 'estado_civil',... but in the form "< form_number>-< field_name>". For example: '0-website' if the field 'website' is in the first form, '1-website' if it is in the second form and so on.
You can print out the whole submitted data dictionary to check that.