Beautiful Soup AttributeError [closed] - web-scraping

Question

Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 2 years ago.
Improve this question
import requests
from bs4 import BeautifulSoup
import pandas as pd
for n in range(1, 16):
response = requests.get(
'https://www.flipkart.com/search?q=books&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as'
'=off '
'&page=' + str(n))
soup = BeautifulSoup(response.text, 'html.parser')
# print(soup.prettify())
urls = list()
for a in soup.find_all('a', {'class': '_2cLu-l'}):
urls.append('https://www.flipkart.com' + a['href'])
# for a in soup.find_all('a', {'class': '_31qSD5'}):
# urls.append('https://www.flipkart.com' + a['href'])
#
# for a in soup.find_all('a', {'class': '_3dqZjq'}):
# urls.append('https://www.flipkart.com' + a['href'])
products = list()
for url in urls:
product = dict()
page_soup = BeautifulSoup(requests.get(url).text, 'html.parser')
name = page_soup.find('h1', {'class': '_9E25nV'})
product['name'] = name.text
price = page_soup.find('div', {'class': '_1vC4OE _3qQ9m1'})
product['price'] = price.text
ratingsAndReviews = page_soup.find('span', {'class': '_38sUEc'})
if ratingsAndReviews is None:
product['ratingsAndReviews'] = '0 ratings & 0 reviews'
else:
product['ratingsAndReviews'] = ratingsAndReviews.text
products.append(product)
df = pd.DataFrame(products)
print(df)
df.to_csv(r'C:\Users\shiva\Desktop\Damn\Output_flipkart.csv', index=True)
product['name'] = name.text | AttributeError: 'NoneType' object has no attribute 'text'

Answer 1

Sometimes the website throws unwanted html in order to block you. So, put a try/except block in inner for loop. In that way even if you some urls don't work, it won't stop the program.
And put the code to convert to dataframe and saving out of the for loops
import requests
from bs4 import BeautifulSoup
import pandas as pd
products = list()
for n in range(1, 16):
response = requests.get(
'https://www.flipkart.com/search?q=books&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as'
'=off '
'&page=' + str(n))
soup = BeautifulSoup(response.text, 'html.parser')
urls = list()
for a in soup.find_all('a', {'class': '_2cLu-l'}):
urls.append('https://www.flipkart.com' + a['href'])
for url in urls:
try:
product = dict()
res = requests.get(url)
page_soup = BeautifulSoup(res.text, 'html.parser')
name = page_soup.find('h1', {'class': '_9E25nV'})
product['name'] = name.text
price = page_soup.find('div', {'class': '_1vC4OE _3qQ9m1'})
product['price'] = price.text
ratingsAndReviews = page_soup.find('span', {'class': '_38sUEc'})
if ratingsAndReviews is None:
product['ratingsAndReviews'] = '0 ratings & 0 reviews'
else:
product['ratingsAndReviews'] = ratingsAndReviews.text
products.append(product)
except Exception as e:
print(e)
df = pd.DataFrame(products)
df.to_csv("data.csv", index=False)