requests works, scrapy.Request does not (why...?) - python-requests

Question

I'm attempting to scrape the first 20 pages of https://en.mzadqatar.com/qatar/cars/sale
The site has an XHR call which checks the origin page of the user before providing the next page.
If a user enters on page 2 the site will revert the user back to the home page.
The desired response can replicated by entering at the above link and then clicking the "NEXT" button at the bottom of the page.
3
If you enter directly through this link https://en.mzadqatar.com/qatar/cars/sale?page=1 the website will revert back to the home page.
I've managed to successfully move through each page with the following code using requests, however I cannot replicate the same response using scrapy.Request. Where am I going wrong.....?
Here is the successful code using the requests library:
import requests
url = "https://en.mzadqatar.com/search"
payload = "type_id=0&id=1&subCategoryId=&pagination=1&search_type=pagination&km_from=&km_to=&price_from=&price_to=&cityId=&CartypeID=&Fueltype=&subsubCategoryId=&gear=&CylinderNumber=&cars_guarantee=&car_condition=&carcolor=&manfactureYear_from=&manfactureYear_to="
headers = {
"cookie": "laravel_session=QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL",
"authority": "en.mzadqatar.com",
"accept": "*/*",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"dnt": "1",
"origin": "https://en.mzadqatar.com",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"x-requested-with": "XMLHttpRequest"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
url = "https://en.mzadqatar.com/qatar/cars/sale"
querystring = {"page":"1"}
payload = ""
headers = {
"cookie": "laravel_session=QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL",
"authority": "en.mzadqatar.com",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"dnt": "1",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}
response = requests.request("GET", url, data=payload, headers=headers, params=querystring)
print(response.text)
Here is the code using scrapy.Requests that reverts back to the homepage
import scrapy
class MzSpider(scrapy.Spider):
name = 'mz'
allowed_domains = ['mzadqatar.com']
start_urls = ['https://en.mzadqatar.com/qatar/cars/sale']
search_url = "https://en.mzadqatar.com/search"
search_body = "type_id=0&id=1&subCategoryId=&pagination=2&search_type=pagination&km_from=&km_to=&price_from=&price_to=&cityId=&CartypeID=&Fueltype=&subsubCategoryId=&gear=&CylinderNumber=&cars_guarantee=&car_condition=&carcolor=&manfactureYear_from=&manfactureYear_to="
dict_search_body = {
"type_id": 0,
"id": 1,
"subCategoryId":"",
"pagination": 1,
"search_type": "pagination",
"km_from": "",
"km_to": "",
"undefined":"" ,
"cityId": "",
"CartypeID":"" ,
"Fueltype": "",
"subsubCategoryId": "",
"gear": "",
"CylinderNumber": "",
"cars_guarantee":"",
"car_condition": "",
"carcolor": "",
"manfactureYear_from":"",
"manfactureYear_to": ""
}
search_headers = {
"authority": "en.mzadqatar.com",
"accept": "*/*",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"dnt": "1",
"origin": "https://en.mzadqatar.com",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"x-requested-with": "XMLHttpRequest"
}
url = "https://en.mzadqatar.com/qatar/cars/sale?page=1"
headers = {
"authority": "en.mzadqatar.com",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"dnt": "1",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}
cookie = {'laravel_session' : 'QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL'}
def search_requests(self):
yield scrapy.Request(
url=self.search_url,
method='POST',
headers=self.search_headers,
body=self.search_body,
cookies=self.cookie,
callback=self.start_requests
)
def start_requests(self):
yield scrapy.Request(
url=self.url,
method='GET',
headers=self.headers,
body="",
cookies=self.cookie,
callback=self.parse
)
def parse(self, response):
print(response.text)
pass
Any help on how to translate requests into scrapy.Requests would be very much appreciated.

Answer 1

As you see you can't get DIFFERENT page results at the same time using SAME cookies. That's why you need to use DIFFERENT cookies for DIFFERENT pages (I use meta={'cookiejar': ...} for that):
import scrapy
class MzSpider(scrapy.Spider):
name = 'mz'
allowed_domains = ['mzadqatar.com']
def start_requests(self):
for page_number in range(1, 10):
payload = {
'type_id': '0',
'id': '1',
'subCategoryId': '',
'pagination': str(page_number),
'search_type': 'pagination',
'km_from': '',
'km_to': '',
'price_from': '',
'price_to': '',
'cityId': '',
'CartypeID': '',
'Fueltype': '',
'subsubCategoryId': '',
'gear': '',
'CylinderNumber': '',
'cars_guarantee': '',
'car_condition': '',
'carcolor': '',
'manfactureYear_from': '',
'manfactureYear_to': '',
}
yield scrapy.FormRequest(
url='https://en.mzadqatar.com/search',
method='POST',
formdata=payload,
headers={
'x-requested-with': 'XMLHttpRequest',
},
cb_kwargs={
'page_number': page_number,
},
meta={'cookiejar': page_number},
callback=self.get_pagination,
)
def get_pagination(self, response, page_number):
yield scrapy.Request(
url=f'https://en.mzadqatar.com/search?page={page_number}',
meta={'cookiejar': page_number},
callback=self.parse_pagination,
cb_kwargs={
'page_number': page_number,
}
)
def parse_pagination(self, response, page_number):
with open(f'Page_{page_number}.html', 'wb') as f:
f.write(response.body)
pass