Unable to scrape all photos even after using selenium to automate scroll - web-scraping

Question

I am a newbie to web scraping. I am currently working on a project where I want to scrape all the photos of an instagram user. The user has 521 posts in total due to which I had used selenium to scroll down till the bottom of the profile. But I am still able to scrape only the first 37 photos. After further inspecting, I found that as the browser scrolls up or down, only the first few rows of img tags are visible in the source code. As I scroll more, the previously visible img tags disappear and only the next rows are visible. So only a certain no. of rows are visible in the html code at any instant. I doubt it to be the reason why I am able to scrape only the first 37 photos.
I want to know how I can scrape all the photos of the profile. Below I have mentioned my current code using Beautiful Soup and Selenium. Here, "scroll_down" function uses selenium to scroll down till the bottom of the profile. I am trying to scrape all the 'img' tags in the function "downloading_images", but as already mentioned, I am able to scrape only first 37 photos.
def downloading_images(self):
soup = BeautifulSoup(self.driver.page_source,'html.parser')
self.all_images = soup.findAll('img')
print(len(self.all_images))
for index,image in enumerate(self.all_images):
filename = "image_" + str(index) + ".jpg"
image_path = os.path.join(self.path,filename)
link = image['src']
print("Downloading image ", index)
response = requests.get(link,stream = True)
try:
with open(image_path,'wb') as file:
shutil.copyfileobj(response.raw,file)
except Exception as e:
print(e)
print('Could not download image no.', index)
print('Image link',link)
def scroll_down(self):
sleep(3)
try:
num_posts = self.driver.find_element_by_xpath('//span[text()[contains(.," posts")]]/span[#class="g47SY "]')
str_num_posts = str(num_posts.text).replace(',','')
self.int_num_posts = int(str_num_posts)
if self.int_num_posts > 12:
num_scrolls = int(self.int_num_posts/12) + 3
print(num_scrolls)
sleep(3)
try:
for win in range(num_scrolls):
print(win)
self.driver.execute_script('window.scrollTo(0,document.body.scrollHeight);')
sleep(3)
except Exception as e:
self.error = True
print(e)
except Exception as e:
self.error = True
print(e)
I searched for all relevant questions here, but none of them could help me understand how I can fetch those images from the code which keeps disappearing upon scrolling.
Hope my question is clear. Thanks in advance.
Edit: Ok, I tried to scrape upon every scroll and it seems to work. Here is my new code.
def downloading_images(self):
print(len(self.all_images))
for index,image in enumerate(self.all_images):
filename = "image_" + str(index) + ".jpg"
image_path = os.path.join(self.path,filename)
link = image['src']
print("Downloading image ", index)
response = requests.get(link,stream = True)
try:
with open(image_path,'wb') as file:
shutil.copyfileobj(response.raw,file)
except Exception as e:
print(e)
print('Could not download image no.', index)
print('Image link',link)
def scroll_down(self):
sleep(3)
try:
num_posts = self.driver.find_element_by_xpath('//span[text()[contains(.," posts")]]/span[#class="g47SY "]')
str_num_posts = str(num_posts.text).replace(',','')
self.int_num_posts = int(str_num_posts)
if self.int_num_posts > 12:
num_scrolls = int(self.int_num_posts/12) + 1
else:
num_scrolls = self.int_num_posts
print(num_scrolls)
sleep(3)
try:
soup = BeautifulSoup(self.driver.page_source,'html.parser')
images = soup.findAll('img')
self.all_images = images
last_height = self.driver.execute_script("return document.body.scrollHeight")
for win in range(num_scrolls):
print(win)
self.driver.execute_script('window.scrollTo(0,document.body.scrollHeight);')
sleep(3)
new_height = self.driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
soup = BeautifulSoup(self.driver.page_source,'html.parser')
images = soup.findAll('img')
self.all_images.extend(images[-12:])
last_height = new_height
print(len(self.all_images))
except Exception as e:
self.error = True
print(e)
except Exception as e:
self.error = True
print(e)
def search_target(self):
try:
search_bar = self.driver.find_element_by_xpath('//input[#class="XTCLo x3qfX "]')
search_bar.send_keys(self.target_username)
taget_profile_url = self.main_url + '/' + self.target_username + '/'
self.driver.get(taget_profile_url)
except Exception as e:
self.error = True
print(e)
def close_notify_box(self):
try:
sleep(3)
not_now_button = self.driver.find_element_by_xpath('//button[#class="aOOlW HoLwm "]')
not_now_button.click()
except Exception:
pass
def log_in(self):
try:
log_in_button = self.driver.find_element_by_xpath('//p[#class="izU2O"]/a')
log_in_button.click()
sleep(3)
user_name_input = self.driver.find_element_by_xpath('//input[#aria-label="Phone number, username, or email"]')
user_name_input.send_keys(self.username)
password_input = self.driver.find_element_by_xpath('//input[#aria-label="Password"]')
password_input.send_keys(self.password)
password_input.submit()
except Exception as e:
self.error = True
print(e)
I would like to know if there are any alternative solutions to this. And whether it is an efficient solution. Thank you.