The script is in data.py and template file is search.mako. The search form is in MainPage method (not included in the code below). I enter the search term but nothing happens. Can you help understand what I am doing wrong? Thank you.
class Pet(db.Model):
name = db.StringProperty()
class Search(webapp.RequestHandler):
def post(self):
query = Pet.all()
results = self.request.get('searchquery')
q = query.filter('name =', 'results')
template_values = {'q': q,}
path = os.path.join(os.path.dirname(__file__), 'search.mako')
templ = Template(filename=path)
self.response.out.write(templ.render(**template_values))
And this is the search.mako
<html>
<body>
% for cat in q:
<p>${cat.name}</p>
% endfor
</html>
</body>
Adding fetch() fixed the problem:
class Search(webapp.RequestHandler):
def post(self):
query = Pet.all()
q = query.filter('name =', self.request.get('searchquery')).fetch(10)
Related
I am doing web-scraping using beautifulsoup in python 3.7. The code below is successfully scraping date, title, tags but not the content of the articles. It is giving None instead.
import time
import requests
from bs4 import BeautifulSoup
from bs4.element import Tag
url = 'https://www.thehindu.com/search/?q=cybersecurity&order=DESC&sort=publishdate&ct=text&page={}'
pages = 32
for page in range(4, pages+1):
res = requests.get(url.format(page))
soup = BeautifulSoup(res.text,"lxml")
for item in soup.find_all("a", {"class": "story-card75x1-text"}, href=True):
_href = item.get("href")
try:
resp = requests.get(_href)
except Exception as e:
try:
resp = requests.get("https://www.thehindu.com"+_href)
except Exception as e:
continue
dateTag = soup.find("span", {"class": "dateline"})
sauce = BeautifulSoup(resp.text,"lxml")
tag = sauce.find("a", {"class": "section-name"})
titleTag = sauce.find("h1", {"class": "title"})
contentTag = sauce.find("div", {"class": "_yeti_done"})
date = None
tagName = None
title = None
content = None
if isinstance(dateTag,Tag):
date = dateTag.get_text().strip()
if isinstance(tag,Tag):
tagName = tag.get_text().strip()
if isinstance(titleTag,Tag):
title = titleTag.get_text().strip()
if isinstance(contentTag,Tag):
content = contentTag.get_text().strip()
print(f'{date}\n {tagName}\n {title}\n {content}\n')
time.sleep(3)
I don't see where is the problem as I am writing the correct class in contentTag.
Thanks.
I guess the links you would like to follow from first page to it's inner page end with .ece. I've applied that logic within the script to traverse those target pages to scrape data from. I've defined selectors for content slightly differently. Now it appears to be working correctly. The following script only scrapes data from page 1. Feel free to change it as per your requirement.
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
url = 'https://www.thehindu.com/search/?q=cybersecurity&order=DESC&sort=publishdate&ct=text&page=1'
base = "https://www.thehindu.com"
res = requests.get(url)
soup = BeautifulSoup(res.text,"lxml")
for item in soup.select(".story-card-news a[href$='.ece']"):
resp = requests.get(urljoin(base,item.get("href")))
sauce = BeautifulSoup(resp.text,"lxml")
title = item.get_text(strip=True)
content = ' '.join([item.get_text(strip=True) for item in sauce.select("[id^='content-body-'] p")])
print(f'{title}\n {content}\n')
I have a problem with copy_relations after publishing a page.
I have a plugin with additional model. This model has ForeignKey to my plugin.
class InfContactForm(CMSPlugin):
name = models.CharField(max_length=50)
def copy_relations(self, oldinstance):
self.inf_contact_form.all().delete()
for inf_contact_form in oldinstance.inf_contact_form.all():
inf_contact_form.pk = None
inf_contact_form.plugin = self
inf_contact_form.save()
class InfContactFormAgreement(models.Model):
inf_contact_form = models.ForeignKey(InfContactForm, related_name="inf_contact_form")
agreement = HTMLField(blank=True, null=True)
The "InfContactFormAgreement" model is then used as stacked inline in "InfContactForm" plugin form.
Like it is written in docs, there is also copy_relations() method but when the page is published, nothing happens. When I get back again to edit mode InfContactFormAgreement is doubled every time I do it.
Here is also my cms_plugins.py file:
class PluginInfContactForm(CMSPluginBase):
render_template = '_contact_form.html'
name = name1
model = InfContactForm
require_parent = False
inlines = [AgreementStackedInline, ]
def render(self, context, instance, placeholder):
context['instance'] = instance
context = super(PluginInfContactForm, self).render(context, instance, placeholder)
agreements = instance.inf_contact_form.all()
context.update({
'agreements': agreements,
})
return context
plugin_pool.register_plugin(PluginInfContactForm)
Thank you for any advice.
instead of:
inf_contact_form.plugin = self
you should use:
inf_contact_form.inf_contact_form = self
and the resulting code should look like this:
def copy_relations(self, oldinstance):
self.inf_contact_form.all().delete()
for inf_contact_form in oldinstance.inf_contact_form.all():
inf_contact_form.pk = None
inf_contact_form.inf_contact_form = self
inf_contact_form.save()
regards
I had a similar situation, the only difference being that my many-to-many relationship was not in a plugin, but an extension (PageExtension).
In my case, #Dariusz solution did not work, and I had to update the implicit "inbetween" model that exists between the Extension model and the Associated model.
My solution (the key being the "through"):
class Extension(PageExtension):
tags_regions = models.ManyToManyField(Region, related_name="articles", blank=True)
def copy_relations(self, oldinstance):
for region_tag in oldinstance.tags_regions.through.objects.filter(extension=oldinstance):
region_tag.extension = self
region_tag.save()
Here is the window I need to enter new password and repeat it again and click 'create'.
My code so far:
createLogin = wait.until(EC.presence_of_element_located((By.XPATH, '//*[#id="Item.MessageUniqueBody"]/div/div/div/div/div[2]/div[2]/a')))
createLogin.click()
time.sleep(10)
try:
newPassword = self.driver.find_elements_by_xpath('//*[#id="editNewUser_newPassword"]')
newPassword1 = self.driver.find_elements_by_xpath('//*[#id="editNewUser_newPasswordRepeat"]')
newPasswordForm = self.driver.find_elements_by_xpath('//*[#id="editNewUserPasswordForm"]/table/tbody/tr[1]/td[1]')
self.driver.switch_to.active_element(newPasswordForm)
time.sleep(3)
newPassword.send_keys('123')
newPassword1.send_keys('123')
time.sleep(2)
# createLog = wait.until(
# EC.presence_of_element_located((By.XPATH, '//*[#id="editNewUserPassword_save"]')))
# createLog.click()
# time.sleep(5)
except NoAlertPresentException as e:
time.sleep(2)
myAccount = wait.until(
EC.presence_of_element_located((By.XPATH, '//*[#id="easMyAccount1"]')))
myAccount.click()
time.sleep(5)
This is the problem.
You are using find_elements_by_xpath rather than find_element_by_xpath
plural vs singular.
find_elements_by_xpath: it gives you a list of web elements with matching identifier.
find_element_by_xpath: it gives you a first web element with matching identifier.
newPassword = self.driver.find_element_by_xpath('//*[#id="editNewUser_newPassword"]')
newPassword1 = self.driver.find_element_by_xpath('//*[#id="editNewUser_newPasswordRepeat"]')
newPasswordForm = self.driver.find_element_by_xpath('//*[#id="editNewUserPasswordForm"]/table/tbody/tr[1]/td[1]')
#gauurang Answer is right, But You have to used find_element_by_xpath, also as your xpath suggested you have id to locate the webelements so it is always better to use id over xpath
Also your xpath are correct
newPassword = self.driver.find_element_by_id('editNewUser_newPassword')
newPassword1 = self.driver.find_element_by_id('editNewUser_newPasswordRepeat')
newPasswordForm = self.driver.find_element_by_xpath('//*[#id="editNewUserPasswordForm"]/table/tbody/tr[1]/td[1]')
I'm pretty new to Rails and trying some basic stuff like conditional classes.
On the 'show' view I have an element that changes styling depending on the stock availability, but also the text changes accordingly.
People keep saying the controller should be as small as possible, but placing this conditional in the view also feels dirty. Is this really the best way?
Current controller:
def show
#tyre = Tyres::Tyre.find_by_id(params[:id])
if #tyre.in_stock
#availability = I18n.t("products.filter.other.in_stock")
#availability_class = 'i-check-circle color--success'
else
#availability = I18n.t("products.filter.other.not_in_stock")
#availability_class = 'i-cross-circle color--important'
end
end
Edit:
Controller:
def show
#tyre = Tyres::Tyre.find_by_id(params[:id])
if #tyre.in_stock
#availability_append = ".in_stock"
else
#availability_append = ".not_in_stock"
end
#availability = I18n.t("products.filter.other#{#availability_append}")
end
View:
.xs-12.description__status
%i{class: (#tyre.in_stock? ? 'i-check-circle color--success' : 'i-cross-circle color--important')}
= #availability
You can clean your controller tyres_controller.rb (i suppose) method,
def show
#tyre = Tyre.find(params[:id]) # I believe you have a model named 'tyre'
end
Then, there will be a file named tyres_helper.rb in your myproject/app/helpers/. Put the following code there,
def tyre_availability(tyre) # it'll return an array with two values, first one is class name, second one is localized value
if tyre.in_stock
return 'i-check-circle color--success', I18n.t("products.filter.other.in_stock")
else
return 'i-cross-circle color--important', I18n.t("products.filter.other.not_in_stock")
end
end
and, in the view you can use,
.xs-12.description__status
%i{:class => tyre_availability(#tyre)[0]}
= tyre_availability(#tyre)[1]
I'm start using Django-MPTT app to get a tree-based approach on my Django-site pages.
For ex. I have pages with sub pages:
Trance:
Vocal Trance(sub page)
Hard Trance(sub page)
Breaks:
Atmo Breaks(sub page)
Progressive Breaks(sub page)
How can I get access to them from urls.py?
What pattern will help?
Do I need to store Full_path in model or it can be done via url pattern?
I assume you mean you want to do URLs like this:
/trance/
/trance/vocal-trance/
/trance/hard-trace/
/breaks/
/breaks/atmo-breaks/
/breaks/progressive-breaks/
If so, it's probably best to store the url fragment in your model. Something like:
from mptt.models import MPTTModel
from django.db import models
from django.template.defaultfilters import slugify
class Page(MPTTModel):
name = models.CharField(max_length=50)
slug = models.CharField(max_length=50,null=True)
url = models.CharField(max_length=255,null=True)
def save(self, *args, **kwargs)
if self.slug is None:
# create a slug that's unique to siblings
slug = slugify(self.name)
self.slug = slug
siblings = self.get_siblings()
i = 1
while siblings.filter(slug=self.slug).exists():
i += 1
self.slug = slug + '-%d' % i
# now create a URL based on parent's url + slug
if self.parent:
self.url = '%s/%s' % (self.parent.url, self.slug)
else:
self.url = self.slug
super(Page, self).save(*args, **kwargs)
Then add a URL pattern:
(r'^pages/(?P<page_url>[\w\d_/-]+)/$', 'pages.views.show_page'),
And in your view you can just fetch the right page:
def show_page(request, page_url=None):
page = get_object_or_404(Page, url=page_url)
...
Thank you for your attention to my problem.
See,How I finally do it.
models.py
class WebPage(MPTTModel):
slug=RuSlugField(max_length=20,unique=True)
title=models.CharField(max_length=50)
content=models.TextField()
parent=TreeForeignKey('self',null=True,blank=True,related_name='children')
class MPTTMeta:
order_insertion_by=['slug']
def get_absolute_url(self):#TODO:: replace with get_ancestors
url = "/%s/" % self.slug
page = self
while page.parent:
url = "/%s%s" % (page.parent.slug,url)
page = page.parent
return url
urls.py
urlpatterns = patterns('website.views',
url(r"^add/$", "add_page",name="add"),
url(r"^(?P<full_slug>.*)/add/$", "add_page",name="add"),
url(r"^(?P<full_slug>.*)/edit/$", "edit_page",name="edit"),
url(r'^$', ListView.as_view(model=WebPage,template_name='index.html',context_object_name="webpages_list",),name='index'),
url(r"^(?P<full_slug>.*)/$", "page", name="page"),
)
views.py
def page(request, full_slug):
# Make a list from full_slug.
# For ex. /trance/progressive_trance/fonarev -> ['trance','progressive_trance','fonarev']
slugs=full_slug.split('/')
page=None
# Get a page by it's slug
if len(slugs)>1:
page=get_object_or_404(WebPage,slug=slugs[-1])#slugs=['trance','vocal_trance'] -> 'vocal_trance'
elif len(slugs)==1:
page=get_object_or_404(WebPage,slug=slugs[0])#slugs=['trance'] -> 'trance'
# Check if page url matches requested full_slug
if page.get_absolute_url().strip('/') == full_slug:
return render_to_response('page.html', {'page': page},context_instance=RequestContext(request))
else:
raise Http404
def edit_page(request,full_slug):
slugs=full_slug.split('/')
page=None
if len(slugs)>1:
page=get_object_or_404(WebPage,slug=slugs[-1])
elif len(slugs)==1:
page=get_object_or_404(WebPage,slug=slugs[0])
if not page.get_absolute_url().strip('/') == full_slug:
raise Http404
# Send POST data for update an existing page.Update a page.
if request.method=='POST':
form=WebPageForm(request.POST, instance=page)
if form.is_valid():
form.save()
return HttpResponseRedirect(page.get_absolute_url())
# Render a form to edit data for existing page
else:
form=WebPageForm(instance=page)
return render_to_response('edit_page.html',{'form':form,},context_instance=RequestContext(request))
def add_page(request,full_slug=None):
parent_page=None
slug=None
if full_slug:
slug=full_slug.split('/')
# If there is a slug in REQUEST(ex.'trance')->we need to add a new_page to her parent_page.
# So get a parent page.
if slug:
if len(slug)>1:
parent_page=get_object_or_404(WebPage,slug=slug[-1])
elif len(slug)==1:
parent_page=get_object_or_404(WebPage,slug=slug[0])
# Create a new_page
if request.method=='POST':
form=WebPageForm(request.POST)
if form.is_valid():
new_page=form.save(commit=False)
if parent_page:
new_page.parent=parent_page
new_page.save()
return HttpResponseRedirect(new_page.get_absolute_url())
# Return an unbounded form
else:
form=WebPageForm()
return render_to_response('add_page.html',{'form':form,},context_instance=RequestContext(request))
The trick is in we have to check if the page really exists accessing to it via full_slug:
if not page.get_absolute_url().strip('/') == full_slug:
raise Http404
Otherwise, it could be wrong allowing to check only by slug.
There's also a django app that will do the work for you: django-mptt-urls
def get_absolute_url(self):
return '/'.join([x['slug'] for x in self.get_ancestors(include_self=True).values()])