I am trying the contents of iFrame and the iFrame doesn't have an src or URL. It has id element. Below is code I am trying to use . Is there anything I am missing here ? Content is empty
await page.goto("https://sites.google.com/view/pinnednote/home", { waitUntil: 'load', timeout: 30000 });
const myFrames = await page.frames();
console.log("Parent IFrame = "+myFrames.length)
for ( x of myFrames) { // Getting all iFrames
try {
const frameElement = await x.frameElement();
const contentFrame = await frameElement.contentFrame();
console.log(await contentFrame.content());
} catch(error){
console.log(error)
}
childs = await x.childFrames();
console.log("Child IFrame = "+childs.length)
for ( y of childs) {
const frameElement = await y.frameElement();
const contentFrame = await frameElement.contentFrame();
console.log(await contentFrame.content());
}
}
Try this:
await page.goto("https://sites.google.com/view/pinnednote/home", { waitUntil: 'load', timeout: 30000 });
const myFrames = await page.frames();
console.log("Parent IFrame = "+myFrames.length)
for ( x of myFrames) { // Getting all iFrames
try {
const frameContent = await x.content();
console.log(frameContent)
} catch(error){
console.log(error)
}
}
You already have the frames in your myFrames array. When you are making the loop you only need to take the content.
Related
I am playing around with Sveltekit and I am struggling a bit..
So my problem is, when I add something to the DB it works, but the new Item does not show in the list until i Refresh the page. My Code looks like:
index.js
import { connectToDatabase } from '$lib/db';
export const post = async ({ request}) => {
const body = await request.json()
console.log(body)
const dbConnection = await connectToDatabase();
const db = dbConnection.db;
const einkaufszettel = db.collection('Einkaufszettel')
await einkaufszettel.insertOne({
name: body.newArticle
});
const einkaufsliste = await einkaufszettel.find().toArray();
return {
status: 200,
body: {
einkaufsliste
}
};
}
export const get = async () => {
const dbConnection = await connectToDatabase();
const db = dbConnection.db;
const einkaufszettel = db.collection('Einkaufszettel')
const einkaufsliste = await einkaufszettel.find().toArray();
console.log(einkaufsliste)
return {
status: 200,
body: {
einkaufsliste,
}
};
}
and the Script of index.svelte
<script>
import Title from '$lib/title.svelte'
export let einkaufsliste = []
let newArticle = ''
const addArticle = async () => {
const res = await fetch('/', {
method: 'POST',
body: JSON.stringify({
newArticle
}),
headers: {
'Content-Type': 'application/json'
}
})
fetchArticles()
}
async function fetchArticles() {
const res = await fetch('/')
console.log(res.body)
}
</script>
In the Network Preview Tab the new Item is already added to the List.
As you can read here, you need to reassign the einkaufsliste after fetching the list of elements from the API.
You can do this in your fetchArticles method, like this:
async function fetchArticles() {
einkaufsliste = await fetch('/')
}
I apologize if I duplicated the topic, but everything I've tried so far doesn't help.
I am trying to scraping some details data from each links from ads.
After opened few links, I get each next time blank page without any data.
Also, if I stop script and start it again I can't do it(because I received blank page on the start) and need to wait few minutes.
This is code and can you please said me where I made mistake.
Thank you :)
const puppeteer = require("puppeteer-extra");
(async () => {
const browser = await puppeteer.launch({
headless: false,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-infobars",
"--window-position=0,0",
"--ignore-certifcate-errors",
"--ignore-certifcate-errors-spki-list",
'--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3312.0 Safari/537.36"',
],
});
const page = await browser.newPage();
await page.goto(
"https://www.ebay-kleinanzeigen.de/s-immobilien/01309/c195l23973"
);
const banner = (await page.$("#gdpr-banner-accept")) !== null;
if (banner) {
await Promise.all([page.click("#gdpr-banner-accept")]);
}
let isBtnDisabled = false;
while (!isBtnDisabled) {
const productsHandles = await page.$$(".ad-listitem.lazyload-item");
for (const producthandle of productsHandles) {
try {
link = await page.evaluate(
(el) => el.querySelector("a.ellipsis").href,
producthandle
);
const eachPage = await browser.newPage();
await eachPage.goto(link, {
waitUntil: "networkidle0",
});
await eachPage.waitForSelector("#viewad-price", {
visible: true,
});
const price = await eachPage.$eval(
"#viewad-price",
(price) => price.textContent
);
console.log(price);
eachPage.close();
} catch (error) {
console.log(error);
}
}
await page.waitForSelector("#srchrslt-pagination > div", { visible: true });
const is_disabled =
(await page.$(
"#srchrslt-pagination > div > div.pagination-nav > .pagination-next"
)) === null;
isBtnDisabled = is_disabled;
if (!is_disabled) {
await Promise.all([
page.click(".pagination-nav > .pagination-next"),
page.waitForNavigation({ waitUntil: "networkidle2" }),
]);
}
}
await browser.close();
})();
I am working on a Blazor application which needs the screen to be shared. I found how to read and display the screen in the Razor page, once it receives the signaled message from the client, which is an Electron application. This is the code for the Electron Application.
const { desktopCapturer } = require('electron')
const signalR = require('#microsoft/signalr')
let connection;
let subject;
let screenCastTimer;
let isStreaming = false;
const framepersecond = 10;
const screenWidth = 1280;
const screenHeight = 800;
async function initializeSignalR() {
connection = new signalR.HubConnectionBuilder()
.withUrl("http://localhost:51064")
.configureLogging(signalR.LogLevel.Information)
.build();
connection.on("NewViewer", function () {
if (isStreaming === false)
startStreamCast()
});
connection.on("NoViewer", function () {
if (isStreaming === true)
stopStreamCast()
});
await connection.start().then(function () {
console.log("connected");
});
return connection;
}
initializeSignalR();
function CaptureScreen() {
return new Promise(function (resolve, reject) {
desktopCapturer.getSources({ type: ['screen'], thumbnailSize: { width: screenWidth, height: screenHeight } },
(error, sources) => {
if (error) console.error(error);
for (const source of sources) {
if (source.name === 'Entire screen') {
resolve(source.thumbnail.toDataURL())
}
}
})
})
}
const agentName = document.getElementById('agentName');
const startCastBtn = document.getElementById('startCast');
const stopCastBtn = document.getElementById('stopCast');
stopCastBtn.setAttribute("disabled", "disabled");
startCastBtn.onclick = function () {
startCastBtn.setAttribute("disabled", "disabled");
stopCastBtn.removeAttribute("disabled");
connection.send("AddScreenCastAgent", agentName.value);
};
function startStreamCast() {
isStreaming = true;
subject = new signalR.Subject();
connection.send("StreamCastData", subject, agentName.value);
screenCastTimer = setInterval(function () {
try {
CaptureScreen().then(function (data) {
subject.next(data);
});
} catch (e) {
console.log(e);
}
}, Math.round(1000 / framepersecond));
}
But I am trying to figure out the equivalent of these lines of code:
const { desktopCapturer } = require('electron')
const signalR = require('#microsoft/signalr')
desktopCapturer.getSources({ type: ['screen'], thumbnailSize: { width: screenWidth, height: screenHeight } },
in an HTML page when signalR.js is used.
Help would be very much appreciated !
I'm trying to crawl a webpage that has a h3 tag under an a tag. I'm getting the a tag just fine, but when trying to get the innerText of h3 I'm getting an undefined value.
This is what I'm trying to crawl:
const puppeteer = require('puppeteer');
const pageURL = "https://producthunt.com";
const webScraping = async pageURL => {
const browser = await puppeteer.launch({
headless: false,
arges: ["--no-sandbox"]
});
const page = await browser.newPage();
let dataObj = {};
try {
await page.goto(pageURL, { waitUntil: 'networkidle2' });
const publishedNews = await page.evaluate(() => {
const newsDOM = document.querySelectorAll("main ul li");
let newsList = [];
newsDOM.forEach(linkElement => {
const text = linkElement.querySelector("a").textContent;
const innerText = linkElement.querySelector("a").innerText;
const url = linkElement.querySelector("a").getAttribute('href');
const title = linkElement.querySelector("h3").innerText;
console.log(title);
newsList.push({
title,
text,
url
});
});
return newsList;
});
dataObj = {
amount: publishedNews.length,
publishedNews
};
} catch (e) {
console.log(e);
}
console.log(dataObj);
browser.close();
return dataObj;
};
webScraping(pageURL).catch(console.error);
Console log works great, but puppeteer throws:
Cannot read property 'innerText' of null
It looks like your solution is working just fine, but you're not controlling whether the h3 tag is null or not. Try adding an if statement before accessing the innerText attribute, or use the code I left below.
const puppeteer = require('puppeteer');
const pageURL = "https://producthunt.com";
const webScraping = async pageURL => {
const browser = await puppeteer.launch({
headless: false,
arges: ["--no-sandbox"]
});
const page = await browser.newPage();
let dataObj = {};
try {
await page.goto(pageURL, { waitUntil: 'networkidle2' });
const publishedNews = await page.evaluate(() => {
let newsList = [];
const newsDOM = document.querySelectorAll("main ul li");
newsDOM.forEach(linkElement => {
const aTag = linkElement.querySelector("a");
const text = aTag.textContent;
const innerText = aTag.innerText;
const url = aTag.getAttribute('href');
let title = aTag.querySelector("h3");
// there may be some <a> without an h3, control
// the null pointer exception here, accessing only
// if title is not 'null'.
if (title) title = title.innerText;
console.log(title);
// changed the object structure to add a key for each attr
newsList.push({
title: title,
text: text,
url: url
});
});
return newsList;
});
// changed the object structure to add a key for the array
dataObj = {
amount: publishedNews.length,
list: publishedNews
};
} catch (e) {
console.log(e);
}
console.log({receivedData: dataObj});
browser.close();
return dataObj;
};
webScraping(pageURL).catch(console.error);
Let me know if this fixes your problem!
I created a firebase function that updates the like count of an comment or post when a new like document is created.
But it throws an 404 error.
exports.updateLikeCount = functions.firestore
.document('likes')
.onCreate((snap, context) => {
const likeObj = snap.data();
if(likeObj.isComment) {
const { _comment } = likeObj;
const commentRef = fstore.collection('comments').doc(_comment);
return commentRef.get()
.then(doc => {
let { likes } = doc.data();
++likes;
return commentRef.update({
likes
});
});
}else {
const { _post } = likeObj;
const postRef = fstore.collection('posts').doc(_post);
return postRef.get()
.then(doc => {
let { likes } = doc.data();
++likes;
return postRef.update({
likes
});
});
}
});
NOTE:
_post and _comment are post ID and comment ID respectively