Handling Pagination with Scrapy - web-scraping

Question

Helo all!
I am trying to scrap this website(https://codeblue.galencentre.org/malaysia/) using Scrapy.
I am having trouble getting the pagination and extracting the titles using the code below. Should I convert 'Content-Type: application/x-www-form-urlencoded' to 'application/json'? I am stuck half way.
Below is my attempt that yields a 'Error 400:HTTP status code is not handled or not allowed'.
import scrapy
from scrapy.http.request.form import FormRequest
class SihatSpider(scrapy.Spider):
name = 'sihat'
def start_requests(self):
query = 'query_params%5Bpost_types%5D=post&query_params%5Bi_attachment%5D=&query_params%5Btaxonomies%5D=category&query_params%5Bmulti_post_types%5D=&query_params%5Bmulti_taxonomies%5D=&query_params%5Bquery_types%5D=0&query_params%5Bi_taxonomies%5D=2%2C+51&query_params%5Be_taxonomies%5D=&query_params%5Bi_ids%5D=&query_params%5Bcq_operator%5D=0&query_params%5Be_ids%5D=&query_params%5Bquery_author%5D=&query_params%5Bquery_offset%5D=&query_params%5Bquery_include_children%5D=0&query_params%5Btoday_post%5D=0&query_params%5Bdatetime_meta%5D=&query_params%5Bwoo_query%5D=0&query_params%5Bpost_count%5D=-1&query_params%5Bposts_per_page%5D=19&query_params%5Bd_i_filter%5D=2&filter=&order=DESC&orderby=date&sub_opt_query%5Bmeta_key_query%5D=&sub_opt_query%5Bpaged%5D=1&sub_opt_query%5Bfirst_query%5D=off&sub_opt_query%5Btotal_pages%5D=181&sub_opt_query%5Bitems_last_page%5D=18&options%5Blayout_style%5D=3&options%5Bgrid_style%5D=0&options%5Blist_style%5D=0&options%5Bcarousel_t_style%5D=0&options%5Bcarousel_f_style%5D=0&options%5Bcreative_style%5D=0&options%5Btimeline_style%5D=0&options%5Bblock_content_style%5D=1&options%5Bgrid_masonry%5D=0&options%5Bshow_arrows%5D=1&options%5Barrows_outside%5D=0&options%5Bshow_dots%5D=1&options%5Binfinite%5D=1&options%5Bautoplay%5D=1&options%5Bautoplayspeed%5D=5000&options%5Bscrollperpage%5D=1&options%5Bspeed%5D=500&options%5Bcentermode%5D=0&options%5Bshow_elements%5D=0&options%5Bav_content%5D=0&options%5Bcc_mobile%5D=0&options%5Bcc_portrait_tablet%5D=0&options%5Bcc_landscape_tablet%5D=0&options%5Bcc_small_desktop%5D=0&options%5Bcc_medium_desktop%5D=0&options%5Bcc_large_desktop%5D=0&options%5Bcc_extra_large_desktop%5D=0&options%5Bimage_size%5D=thumbnail&options%5Bimage_size_s%5D=thumbnail&options%5Bs_image%5D=1&options%5Bs_image_link%5D=1&options%5Bs_image_link_target%5D=0&options%5Bs_icon_lightbox_video%5D=0&options%5Bvideo_url_meta%5D=0&options%5Bvideo_url_meta_key%5D=&options%5Bs_icon_lightbox_image%5D=0&options%5Bs_icon_link%5D=0&options%5Bs_icon_link_target%5D=0&options%5Bs_image_hover_effect%5D=1&options%5Bs_overlay_hover_effect%5D=&options%5Bs_overlay_settings%5D=0&options%5Bs_overlay_color%5D=&options%5Bs_image_post_format%5D=0&options%5Bs_image_post_format_pos%5D=ul-bottom-right&options%5Bs_title%5D=1&options%5Bs_title_limit%5D=0&options%5Bs_title_link%5D=1&options%5Bs_title_link_target%5D=0&options%5Bs_excerpt%5D=1&options%5Bs_excerpt_rbtn%5D=0&options%5Bs_excerpt_f%5D=get_the_excerpt&options%5Bs_excerpt_sc%5D=1&options%5Bs_excerpt_sh%5D=1&options%5Bs_excerpt_length%5D=0&options%5Bs_categories%5D=0&options%5Bs_s_categories%5D=0&options%5Bs_s_categories_parent%5D=0&options%5Bex_items_taxonomies%5D=&options%5Bs_c_categories%5D=0&options%5Bs_ct_categories%5D=&options%5Bs_cb_categories%5D=&options%5Bs_categories_target%5D=0&options%5Bs_metas_o%5D=1&options%5Bs_metas_o_author%5D=0&options%5Bs_metas_o_author_avatar%5D=0&options%5Bs_metas_o_time%5D=1&options%5Btime_format%5D=j+F%2C+Y&options%5Bs_metas_o_comment%5D=0&options%5Bs_metas_o_like%5D=0&options%5Bs_metas_o_share%5D=0&options%5Bcustom_meta_o%5D=&options%5Bs_metas_t%5D=0&options%5Bs_metas_t_author%5D=0&options%5Bs_metas_t_author_avatar%5D=0&options%5Bs_metas_t_time%5D=0&options%5Btime_format_t%5D=F+j%2C+Y&options%5Bs_metas_t_comment%5D=0&options%5Bs_metas_t_like%5D=1&options%5Bs_metas_t_share%5D=1&options%5Bcustom_meta_t%5D=&options%5Bs_metas_t_readmore%5D=1&options%5Bs_metas_t_readmore_link_target%5D=0&options%5Bshare_text%5D=&options%5Bread_more_text%5D=&options%5Bbefore_author_text%5D=&options%5Bpagination%5D=0&options%5Bloadmore_text%5D=&options%5Bprev_text%5D=&options%5Bnext_text%5D=&options%5Banimate%5D=default&options%5Blazyload%5D=0&options%5Blazyload_p%5D=&options%5Bgeodirectory_rating%5D=0&options%5Bquick_view%5D=1&options%5Bquick_view_mode%5D=0&options%5Bextra_class%5D=&options%5Brnd_id%5D=ul93310&options%5Bs_title_small%5D=1&options%5Bs_title_limit_small%5D=0&options%5Bs_title_link_small%5D=1&options%5Bs_title_link_target_small%5D=0&options%5Bs_categories_small%5D=0&options%5Bs_s_categories_small%5D=0&options%5Bs_s_categories_parent_small%5D=0&options%5Bex_items_taxonomies_small%5D=&options%5Bs_c_categories_small%5D=0&options%5Bs_ct_categories_small%5D=&options%5Bs_cb_categories_small%5D=&options%5Bs_categories_target_small%5D=0&options%5Bs_metas_o_small%5D=1&options%5Bs_metas_o_author_small%5D=0&options%5Bs_metas_o_author_avatar_small%5D=0&options%5Bs_metas_o_time_small%5D=1&options%5Btime_format_small%5D=j+F%2C+Y&options%5Bs_metas_o_comment_small%5D=0&options%5Bs_metas_o_like_small%5D=0&options%5Bs_metas_o_share_small%5D=0&options%5Bcustom_meta_o_small%5D=&options%5Bwoo_show_price%5D=0&options%5Bwoo_show_rating%5D=0&options%5Bwoo_show_cart%5D=0&options%5Bqv_s_title%5D=1&options%5Bqv_s_categories%5D=1&options%5Bqv_s_s_categories%5D=0&options%5Bqv_s_s_categories_parent%5D=0&options%5Bqv_ex_items_taxonomies%5D=&options%5Bqv_s_c_categories%5D=0&options%5Bqv_s_ct_categories%5D=&options%5Bqv_s_cb_categories%5D=&options%5Bqv_s_categories_target%5D=0&options%5Bqv_s_metas_o%5D=1&options%5Bqv_s_metas_o_author%5D=1&options%5Bqv_s_metas_o_author_avatar%5D=0&options%5Bqv_s_metas_o_time%5D=1&options%5Bqv_time_format%5D=F+j%2C+Y&options%5Bqv_s_metas_o_comment%5D=1&options%5Bqv_s_metas_o_like%5D=1&options%5Bqv_custom_meta_o%5D=&options%5Bqv_show_content%5D=1&options%5Bqv_content_stripsc%5D=0&options%5Bqv_show_share%5D=1&options%5Bqv_woo_show_rating%5D=1&options%5Bqv_s_featured_image%5D=1&options%5Bgoo_ads_client%5D=&options%5Bgoo_ads_id%5D=&options%5Bgoo_ads_offset%5D=1&options%5Bcss_class%5D=&random_id=ul93310&action=ultimatelayoutsajaxaction'
yield scrapy.FormRequest(
url = 'https://codeblue.galencentre.org/wp-admin/admin-ajax.php',
method='POST',
body=query,
headers={
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-requested-with': 'XMLHttpRequest'
},
callback=self.parse,
meta = {
'paged':1
}
)
def parse(self, response):
current_page = response.meta['paged']
for text in response.xpath("//div[#class='ultimate-layouts-content entry']"):
yield {
'date': text.xpath("./div[#class='ultimate-layouts-metas posted-on ']/div[#class='ultimate-layouts-metas-wrap']/div[#data-class='ul-time-metas']/span/text()").get(),
'title': text.xpath("./h3/a[#class='ultimate-layouts-title-link']/text()").get()
}
count = 181
current_page = 1
if current_page <= count:
current_page += 1
yield FormRequest(
url = 'https://codeblue.galencentre.org/wp-admin/admin-ajax.php',
method='POST',
body='query_params%5Bpost_types%5D=post&query_params%5Bi_attachment%5D=&query_params%5Btaxonomies%5D=category&query_params%5Bmulti_post_types%5D=&query_params%5Bmulti_taxonomies%5D=&query_params%5Bquery_types%5D=0&query_params%5Bi_taxonomies%5D=2%2C+51&query_params%5Be_taxonomies%5D=&query_params%5Bi_ids%5D=&query_params%5Bcq_operator%5D=0&query_params%5Be_ids%5D=&query_params%5Bquery_author%5D=&query_params%5Bquery_offset%5D=&query_params%5Bquery_include_children%5D=0&query_params%5Btoday_post%5D=0&query_params%5Bdatetime_meta%5D=&query_params%5Bwoo_query%5D=0&query_params%5Bpost_count%5D=-1&query_params%5Bposts_per_page%5D=19&query_params%5Bd_i_filter%5D=2&filter=&order=DESC&orderby=date&sub_opt_query%5Bmeta_key_query%5D=&sub_opt_query%5Bpaged%5D={current_page}&sub_opt_query%5Bfirst_query%5D=off&sub_opt_query%5Btotal_pages%5D=181&sub_opt_query%5Bitems_last_page%5D=18&options%5Blayout_style%5D=3&options%5Bgrid_style%5D=0&options%5Blist_style%5D=0&options%5Bcarousel_t_style%5D=0&options%5Bcarousel_f_style%5D=0&options%5Bcreative_style%5D=0&options%5Btimeline_style%5D=0&options%5Bblock_content_style%5D=1&options%5Bgrid_masonry%5D=0&options%5Bshow_arrows%5D=1&options%5Barrows_outside%5D=0&options%5Bshow_dots%5D=1&options%5Binfinite%5D=1&options%5Bautoplay%5D=1&options%5Bautoplayspeed%5D=5000&options%5Bscrollperpage%5D=1&options%5Bspeed%5D=500&options%5Bcentermode%5D=0&options%5Bshow_elements%5D=0&options%5Bav_content%5D=0&options%5Bcc_mobile%5D=0&options%5Bcc_portrait_tablet%5D=0&options%5Bcc_landscape_tablet%5D=0&options%5Bcc_small_desktop%5D=0&options%5Bcc_medium_desktop%5D=0&options%5Bcc_large_desktop%5D=0&options%5Bcc_extra_large_desktop%5D=0&options%5Bimage_size%5D=thumbnail&options%5Bimage_size_s%5D=thumbnail&options%5Bs_image%5D=1&options%5Bs_image_link%5D=1&options%5Bs_image_link_target%5D=0&options%5Bs_icon_lightbox_video%5D=0&options%5Bvideo_url_meta%5D=0&options%5Bvideo_url_meta_key%5D=&options%5Bs_icon_lightbox_image%5D=0&options%5Bs_icon_link%5D=0&options%5Bs_icon_link_target%5D=0&options%5Bs_image_hover_effect%5D=1&options%5Bs_overlay_hover_effect%5D=&options%5Bs_overlay_settings%5D=0&options%5Bs_overlay_color%5D=&options%5Bs_image_post_format%5D=0&options%5Bs_image_post_format_pos%5D=ul-bottom-right&options%5Bs_title%5D=1&options%5Bs_title_limit%5D=0&options%5Bs_title_link%5D=1&options%5Bs_title_link_target%5D=0&options%5Bs_excerpt%5D=1&options%5Bs_excerpt_rbtn%5D=0&options%5Bs_excerpt_f%5D=get_the_excerpt&options%5Bs_excerpt_sc%5D=1&options%5Bs_excerpt_sh%5D=1&options%5Bs_excerpt_length%5D=0&options%5Bs_categories%5D=0&options%5Bs_s_categories%5D=0&options%5Bs_s_categories_parent%5D=0&options%5Bex_items_taxonomies%5D=&options%5Bs_c_categories%5D=0&options%5Bs_ct_categories%5D=&options%5Bs_cb_categories%5D=&options%5Bs_categories_target%5D=0&options%5Bs_metas_o%5D=1&options%5Bs_metas_o_author%5D=0&options%5Bs_metas_o_author_avatar%5D=0&options%5Bs_metas_o_time%5D=1&options%5Btime_format%5D=j+F%2C+Y&options%5Bs_metas_o_comment%5D=0&options%5Bs_metas_o_like%5D=0&options%5Bs_metas_o_share%5D=0&options%5Bcustom_meta_o%5D=&options%5Bs_metas_t%5D=0&options%5Bs_metas_t_author%5D=0&options%5Bs_metas_t_author_avatar%5D=0&options%5Bs_metas_t_time%5D=0&options%5Btime_format_t%5D=F+j%2C+Y&options%5Bs_metas_t_comment%5D=0&options%5Bs_metas_t_like%5D=1&options%5Bs_metas_t_share%5D=1&options%5Bcustom_meta_t%5D=&options%5Bs_metas_t_readmore%5D=1&options%5Bs_metas_t_readmore_link_target%5D=0&options%5Bshare_text%5D=&options%5Bread_more_text%5D=&options%5Bbefore_author_text%5D=&options%5Bpagination%5D=0&options%5Bloadmore_text%5D=&options%5Bprev_text%5D=&options%5Bnext_text%5D=&options%5Banimate%5D=default&options%5Blazyload%5D=0&options%5Blazyload_p%5D=&options%5Bgeodirectory_rating%5D=0&options%5Bquick_view%5D=1&options%5Bquick_view_mode%5D=0&options%5Bextra_class%5D=&options%5Brnd_id%5D=ul93310&options%5Bs_title_small%5D=1&options%5Bs_title_limit_small%5D=0&options%5Bs_title_link_small%5D=1&options%5Bs_title_link_target_small%5D=0&options%5Bs_categories_small%5D=0&options%5Bs_s_categories_small%5D=0&options%5Bs_s_categories_parent_small%5D=0&options%5Bex_items_taxonomies_small%5D=&options%5Bs_c_categories_small%5D=0&options%5Bs_ct_categories_small%5D=&options%5Bs_cb_categories_small%5D=&options%5Bs_categories_target_small%5D=0&options%5Bs_metas_o_small%5D=1&options%5Bs_metas_o_author_small%5D=0&options%5Bs_metas_o_author_avatar_small%5D=0&options%5Bs_metas_o_time_small%5D=1&options%5Btime_format_small%5D=j+F%2C+Y&options%5Bs_metas_o_comment_small%5D=0&options%5Bs_metas_o_like_small%5D=0&options%5Bs_metas_o_share_small%5D=0&options%5Bcustom_meta_o_small%5D=&options%5Bwoo_show_price%5D=0&options%5Bwoo_show_rating%5D=0&options%5Bwoo_show_cart%5D=0&options%5Bqv_s_title%5D=1&options%5Bqv_s_categories%5D=1&options%5Bqv_s_s_categories%5D=0&options%5Bqv_s_s_categories_parent%5D=0&options%5Bqv_ex_items_taxonomies%5D=&options%5Bqv_s_c_categories%5D=0&options%5Bqv_s_ct_categories%5D=&options%5Bqv_s_cb_categories%5D=&options%5Bqv_s_categories_target%5D=0&options%5Bqv_s_metas_o%5D=1&options%5Bqv_s_metas_o_author%5D=1&options%5Bqv_s_metas_o_author_avatar%5D=0&options%5Bqv_s_metas_o_time%5D=1&options%5Bqv_time_format%5D=F+j%2C+Y&options%5Bqv_s_metas_o_comment%5D=1&options%5Bqv_s_metas_o_like%5D=1&options%5Bqv_custom_meta_o%5D=&options%5Bqv_show_content%5D=1&options%5Bqv_content_stripsc%5D=0&options%5Bqv_show_share%5D=1&options%5Bqv_woo_show_rating%5D=1&options%5Bqv_s_featured_image%5D=1&options%5Bgoo_ads_client%5D=&options%5Bgoo_ads_id%5D=&options%5Bgoo_ads_offset%5D=1&options%5Bcss_class%5D=&random_id=ul93310&action=ultimatelayoutsajaxaction',
headers={
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-requested-with': 'XMLHttpRequest'
},
callback=self.parse,
meta = {
'paged':current_page
}
)

Answer 1

It looks like you're not actually getting to the next page at all.
All you have really done is set up an integer to count from 1 to 183, without a direct execution to changing the pages.
One way of doing what you want is to checkout the payload.
This line here:
sub_opt_query[paged]: 2
gives the page number, so you just include the entire payload removing those without any values and assorting everything as strings.
With this in mind, we get the following scraper that get's all the pages respectively:
import scrapy
from scrapy.http.request.form import FormRequest
from scrapy.crawler import CrawlerProcess
class SihatSpider(scrapy.Spider):
name = 'sihat'
def start_requests(self):
query = 'query_params%5Bpost_types%5D=post&query_params%5Bi_attachment%5D=&query_params%5Btaxonomies%5D=category&query_params%5Bmulti_post_types%5D=&query_params%5Bmulti_taxonomies%5D=&query_params%5Bquery_types%5D=0&query_params%5Bi_taxonomies%5D=2%2C+51&query_params%5Be_taxonomies%5D=&query_params%5Bi_ids%5D=&query_params%5Bcq_operator%5D=0&query_params%5Be_ids%5D=&query_params%5Bquery_author%5D=&query_params%5Bquery_offset%5D=&query_params%5Bquery_include_children%5D=0&query_params%5Btoday_post%5D=0&query_params%5Bdatetime_meta%5D=&query_params%5Bwoo_query%5D=0&query_params%5Bpost_count%5D=-1&query_params%5Bposts_per_page%5D=19&query_params%5Bd_i_filter%5D=2&filter=&order=DESC&orderby=date&sub_opt_query%5Bmeta_key_query%5D=&sub_opt_query%5Bpaged%5D=1&sub_opt_query%5Bfirst_query%5D=off&sub_opt_query%5Btotal_pages%5D=181&sub_opt_query%5Bitems_last_page%5D=18&options%5Blayout_style%5D=3&options%5Bgrid_style%5D=0&options%5Blist_style%5D=0&options%5Bcarousel_t_style%5D=0&options%5Bcarousel_f_style%5D=0&options%5Bcreative_style%5D=0&options%5Btimeline_style%5D=0&options%5Bblock_content_style%5D=1&options%5Bgrid_masonry%5D=0&options%5Bshow_arrows%5D=1&options%5Barrows_outside%5D=0&options%5Bshow_dots%5D=1&options%5Binfinite%5D=1&options%5Bautoplay%5D=1&options%5Bautoplayspeed%5D=5000&options%5Bscrollperpage%5D=1&options%5Bspeed%5D=500&options%5Bcentermode%5D=0&options%5Bshow_elements%5D=0&options%5Bav_content%5D=0&options%5Bcc_mobile%5D=0&options%5Bcc_portrait_tablet%5D=0&options%5Bcc_landscape_tablet%5D=0&options%5Bcc_small_desktop%5D=0&options%5Bcc_medium_desktop%5D=0&options%5Bcc_large_desktop%5D=0&options%5Bcc_extra_large_desktop%5D=0&options%5Bimage_size%5D=thumbnail&options%5Bimage_size_s%5D=thumbnail&options%5Bs_image%5D=1&options%5Bs_image_link%5D=1&options%5Bs_image_link_target%5D=0&options%5Bs_icon_lightbox_video%5D=0&options%5Bvideo_url_meta%5D=0&options%5Bvideo_url_meta_key%5D=&options%5Bs_icon_lightbox_image%5D=0&options%5Bs_icon_link%5D=0&options%5Bs_icon_link_target%5D=0&options%5Bs_image_hover_effect%5D=1&options%5Bs_overlay_hover_effect%5D=&options%5Bs_overlay_settings%5D=0&options%5Bs_overlay_color%5D=&options%5Bs_image_post_format%5D=0&options%5Bs_image_post_format_pos%5D=ul-bottom-right&options%5Bs_title%5D=1&options%5Bs_title_limit%5D=0&options%5Bs_title_link%5D=1&options%5Bs_title_link_target%5D=0&options%5Bs_excerpt%5D=1&options%5Bs_excerpt_rbtn%5D=0&options%5Bs_excerpt_f%5D=get_the_excerpt&options%5Bs_excerpt_sc%5D=1&options%5Bs_excerpt_sh%5D=1&options%5Bs_excerpt_length%5D=0&options%5Bs_categories%5D=0&options%5Bs_s_categories%5D=0&options%5Bs_s_categories_parent%5D=0&options%5Bex_items_taxonomies%5D=&options%5Bs_c_categories%5D=0&options%5Bs_ct_categories%5D=&options%5Bs_cb_categories%5D=&options%5Bs_categories_target%5D=0&options%5Bs_metas_o%5D=1&options%5Bs_metas_o_author%5D=0&options%5Bs_metas_o_author_avatar%5D=0&options%5Bs_metas_o_time%5D=1&options%5Btime_format%5D=j+F%2C+Y&options%5Bs_metas_o_comment%5D=0&options%5Bs_metas_o_like%5D=0&options%5Bs_metas_o_share%5D=0&options%5Bcustom_meta_o%5D=&options%5Bs_metas_t%5D=0&options%5Bs_metas_t_author%5D=0&options%5Bs_metas_t_author_avatar%5D=0&options%5Bs_metas_t_time%5D=0&options%5Btime_format_t%5D=F+j%2C+Y&options%5Bs_metas_t_comment%5D=0&options%5Bs_metas_t_like%5D=1&options%5Bs_metas_t_share%5D=1&options%5Bcustom_meta_t%5D=&options%5Bs_metas_t_readmore%5D=1&options%5Bs_metas_t_readmore_link_target%5D=0&options%5Bshare_text%5D=&options%5Bread_more_text%5D=&options%5Bbefore_author_text%5D=&options%5Bpagination%5D=0&options%5Bloadmore_text%5D=&options%5Bprev_text%5D=&options%5Bnext_text%5D=&options%5Banimate%5D=default&options%5Blazyload%5D=0&options%5Blazyload_p%5D=&options%5Bgeodirectory_rating%5D=0&options%5Bquick_view%5D=1&options%5Bquick_view_mode%5D=0&options%5Bextra_class%5D=&options%5Brnd_id%5D=ul93310&options%5Bs_title_small%5D=1&options%5Bs_title_limit_small%5D=0&options%5Bs_title_link_small%5D=1&options%5Bs_title_link_target_small%5D=0&options%5Bs_categories_small%5D=0&options%5Bs_s_categories_small%5D=0&options%5Bs_s_categories_parent_small%5D=0&options%5Bex_items_taxonomies_small%5D=&options%5Bs_c_categories_small%5D=0&options%5Bs_ct_categories_small%5D=&options%5Bs_cb_categories_small%5D=&options%5Bs_categories_target_small%5D=0&options%5Bs_metas_o_small%5D=1&options%5Bs_metas_o_author_small%5D=0&options%5Bs_metas_o_author_avatar_small%5D=0&options%5Bs_metas_o_time_small%5D=1&options%5Btime_format_small%5D=j+F%2C+Y&options%5Bs_metas_o_comment_small%5D=0&options%5Bs_metas_o_like_small%5D=0&options%5Bs_metas_o_share_small%5D=0&options%5Bcustom_meta_o_small%5D=&options%5Bwoo_show_price%5D=0&options%5Bwoo_show_rating%5D=0&options%5Bwoo_show_cart%5D=0&options%5Bqv_s_title%5D=1&options%5Bqv_s_categories%5D=1&options%5Bqv_s_s_categories%5D=0&options%5Bqv_s_s_categories_parent%5D=0&options%5Bqv_ex_items_taxonomies%5D=&options%5Bqv_s_c_categories%5D=0&options%5Bqv_s_ct_categories%5D=&options%5Bqv_s_cb_categories%5D=&options%5Bqv_s_categories_target%5D=0&options%5Bqv_s_metas_o%5D=1&options%5Bqv_s_metas_o_author%5D=1&options%5Bqv_s_metas_o_author_avatar%5D=0&options%5Bqv_s_metas_o_time%5D=1&options%5Bqv_time_format%5D=F+j%2C+Y&options%5Bqv_s_metas_o_comment%5D=1&options%5Bqv_s_metas_o_like%5D=1&options%5Bqv_custom_meta_o%5D=&options%5Bqv_show_content%5D=1&options%5Bqv_content_stripsc%5D=0&options%5Bqv_show_share%5D=1&options%5Bqv_woo_show_rating%5D=1&options%5Bqv_s_featured_image%5D=1&options%5Bgoo_ads_client%5D=&options%5Bgoo_ads_id%5D=&options%5Bgoo_ads_offset%5D=1&options%5Bcss_class%5D=&random_id=ul93310&action=ultimatelayoutsajaxaction'
for i in range(1, 183, 1):
yield scrapy.FormRequest(
url = 'https://codeblue.galencentre.org/wp-admin/admin-ajax.php',
method='POST',
body=query,
headers={
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-requested-with': 'XMLHttpRequest'
},
formdata = {'query_params[post_types]': 'post',
'query_params[taxonomies]': 'category',
'query_params[query_types]': '0',
'query_params[cq_operator]': '0',
'query_params[query_include_children]': '0',
'query_params[today_post]': '0',
'query_params[woo_query]': '0',
'query_params[post_count]': '-1',
'query_params[posts_per_page]': '19',
'query_params[d_i_filter]': '2',
'order': 'DESC',
'orderby': 'date',
'sub_opt_query[paged]': str(i),
'sub_opt_query[first_query]': 'off',
'sub_opt_query[total_pages]': '183',
'sub_opt_query[items_last_page]': '2',
'options[layout_style]': '3',
'options[grid_style]': '0',
'options[list_style]': '0',
'options[carousel_t_style]': '0',
'options[carousel_f_style]': '0',
'options[creative_style]': '0',
'options[timeline_style]': '0',
'options[block_content_style]': '1',
'options[grid_masonry]': '0',
'options[show_arrows]': '1',
'options[arrows_outside]': '0',
'options[show_dots]': '1',
'options[infinite]': '1',
'options[autoplay]': '1',
'options[autoplayspeed]': '5000',
'options[scrollperpage]': '1',
'options[speed]': '500',
'options[centermode]': '0',
'options[show_elements]': '0',
'options[av_content]': '0',
'options[cc_mobile]': '0',
'options[cc_portrait_tablet]': '0',
'options[cc_landscape_tablet]': '0',
'options[cc_small_desktop]': '0',
'options[cc_medium_desktop]': '0',
'options[cc_large_desktop]': '0',
'options[cc_extra_large_desktop]': '0',
'options[image_size]': 'thumbnail',
'options[image_size_s]': 'thumbnail',
'options[s_image]': '1',
'options[s_image_link]': '1',
'options[s_image_link_target]': '0',
'options[s_icon_lightbox_video]': '0',
'options[video_url_meta]': '0',
'options[s_icon_lightbox_image]': '0',
'options[s_icon_link]': '0',
'options[s_icon_link_target]': '0',
'options[s_image_hover_effect]': '1',
'options[s_overlay_settings]': '0',
'options[s_image_post_format]': '0',
'options[s_image_post_format_pos]': 'ul-bottom-right',
'options[s_title]': '1',
'options[s_title_limit]': '0',
'options[s_title_link]': '1',
'options[s_title_link_target]': '0',
'options[s_excerpt]': '1',
'options[s_excerpt_rbtn]': '0',
'options[s_excerpt_f]': 'get_the_excerpt',
'options[s_excerpt_sc]': '1',
'options[s_excerpt_sh]': '1',
'options[s_excerpt_length]': '0',
'options[s_categories]': '0',
'options[s_s_categories]': '0',
'options[s_s_categories_parent]': '0',
'options[s_c_categories]': '0',
'options[s_categories_target]': '0',
'options[s_metas_o]': '1',
'options[s_metas_o_author]': '0',
'options[s_metas_o_author_avatar]': '0',
'options[s_metas_o_time]': '1',
'options[time_format]': 'j F, Y',
'options[s_metas_o_comment]': '0',
'options[s_metas_o_like]': '0',
'options[s_metas_o_share]': '0',
'options[s_metas_t]': '0',
'options[s_metas_t_author]': '0',
'options[s_metas_t_author_avatar]': '0',
'options[s_metas_t_time]': '0',
'options[time_format_t]': 'F j, Y',
'options[s_metas_t_comment]': '0',
'options[s_metas_t_like]': '1',
'options[s_metas_t_share]': '1',
'options[s_metas_t_readmore]': '1',
'options[s_metas_t_readmore_link_target]': '0',
'options[pagination]': '0',
'options[animate]': 'default',
'options[lazyload]': '0',
'options[geodirectory_rating]': '0',
'options[quick_view]': '1',
'options[quick_view_mode]': '0',
'options[rnd_id]': 'ul32943',
'options[s_title_small]': '1',
'options[s_title_limit_small]': '0',
'options[s_title_link_small]': '1',
'options[s_title_link_target_small]': '0',
'options[s_categories_small]': '0',
'options[s_s_categories_small]': '0',
'options[s_s_categories_parent_small]': '0',
'options[s_c_categories_small]': '0',
'options[s_categories_target_small]': '0',
'options[s_metas_o_small]': '1',
'options[s_metas_o_author_small]': '0',
'options[s_metas_o_author_avatar_small]': '0',
'options[s_metas_o_time_small]': '1',
'options[time_format_small]': 'j F, Y',
'options[s_metas_o_comment_small]': '0',
'options[s_metas_o_like_small]': '0',
'options[s_metas_o_share_small]': '0',
'options[woo_show_price]': '0',
'options[woo_show_rating]': '0',
'options[woo_show_cart]': '0',
'options[qv_s_title]': '1',
'options[qv_s_categories]': '1',
'options[qv_s_s_categories]': '0',
'options[qv_s_s_categories_parent]': '0',
'options[qv_s_c_categories]': '0',
'options[qv_s_categories_target]': '0',
'options[qv_s_metas_o]': '1',
'options[qv_s_metas_o_author]': '1',
'options[qv_s_metas_o_author_avatar]': '0',
'options[qv_s_metas_o_time]': '1',
'options[qv_time_format]': 'F j, Y',
'options[qv_s_metas_o_comment]': '1',
'options[qv_s_metas_o_like]': '1',
'options[qv_show_content]': '1',
'options[qv_content_stripsc]': '0',
'options[qv_show_share]': '1',
'options[qv_woo_show_rating]': '1',
'options[qv_s_featured_image]': '1',
'options[goo_ads_offset]': '1',
'random_id': 'ul32943',
'action': 'ultimatelayoutsajaxaction'},
callback=self.parse,
)
def parse(self, response):
for text in response.xpath("//div[#class='ultimate-layouts-content entry']"):
yield {
'date': text.xpath("./div[#class='ultimate-layouts-metas posted-on ']/div[#class='ultimate-layouts-metas-wrap']/div[#data-class='ul-time-metas']/span/text()").get(),
'title': text.xpath("./h3/a[#class='ultimate-layouts-title-link']/text()").get()
}
process = CrawlerProcess(
settings={
'FEED_URI': 'Media_content.jl',
'FEED_FORMAT': 'jsonlines'
}
)
process.crawl(SihatSpider)
process.start()