I'm experimenting with gjs and webkit2, how can i get the http headers of a request made with load_uri
i have the following code
const Gtk = imports.gi.Gtk, WebKit=imports.gi.WebKit2, contentManager=new WebKit.UserContentManager,
view = WebKit.WebView.new_with_user_content_manager(contentManager);
Gtk.init(null);
let win = new Gtk.Window(), Response=new WebKit.URIResponse();
contentManager.add_script (new WebKit.UserScript("alert ('test');",0,1,null,null));
view.load_uri('https://www.gnome.org');
win.add(view);
win.set_title("test");
win.set_icon_from_file("/games/aptdaemon-resolve.png");
win.connect('destroy', () => { Gtk.main_quit(); });
win.set_size_request(640, 480);
win.show_all();
view.connect("load-changed",function (instance,state)
{
if (state == 3)
{
log ("URL"+Response.get_uri());
view.run_javascript ("alert (document.body.innerHTML)",null,null);
}
});
Gtk.main();
for example Response.get_uri returns an empty string, how to access response headers, and how to exchange messages between scripts injected with view.run_javascript and gjs. i want the body html be sent to gjs-?
got it
const Gtk = imports.gi.Gtk;
const WebKit=imports.gi.WebKit2;
Gtk.init(null);
const win = new Gtk.Window(), contentManager=new WebKit.UserContentManager, view = WebKit.WebView.new_with_user_content_manager(contentManager);
let response_STR;
contentManager.connect("script-message-received::pipe", function (instance, message)
{
message=message.get_js_value().to_string ();
log (message);
});
contentManager.register_script_message_handler("pipe");
view.load_uri('https://www.gnome.org');
win.add(view);
win.set_title("test");
win.connect('destroy', () => { Gtk.main_quit(); });
win.set_size_request(640, 480);
win.show_all();
view.connect("load-changed",function (instance,status)
{
let headers, response_STR="";
if (status == 3)
{
/* WebKitView.get_main_resource -> returns WebResource
WebResource.get_response -> returns URIResponse
URIResponse.get_http_headers -> returns Soup.MessageHeaders */
headers=view.get_main_resource().get_response().get_http_headers();
response_STR="";
headers.foreach ((name, value) => { response_STR+=name+": "+value+"\n"});
view.run_javascript('window.webkit.messageHandlers.pipe.postMessage(document.body.innerHTML);', null, null);
log (response_STR);
}
});
Gtk.main();
Related
I'm using npm ws package for websocket. I'm not being able send message into chatroom. any idea how to do it. Below is my code with sendMessage() and broadcast()
const sendMessage = (room_name, message, socket) => {
// rooms[room_name].message = JSON.stringify(message);
const obj = rooms[room_name];
for(i=0;i<obj.length;i++){
var temp = obj[i];
for(var innerObject in temp){
var wsClientID = temp[innerObject];
if(socket!==wsClientID){
wsClientID.send(JSON.stringify({
'message':message,
}));
}
}
}
// rooms[room_name].message = message;
socket.send(JSON.stringify(message));
// rooms[room_name].message = message;
}
socket.on("message", async (data) => {
broadcast(data)}
function broadcast(data) {
var count = 0;
for (const client of server.clients) {
if (client.readyState === socket.OPEN) {
count++;
client.send(data.toString())
}
}
}
I'm using Node/Puppeteer in the code below, passing in a large list of URL's for traversal and scraping. It has been difficult to do it asynchronously, though I find that I am getting closer and closer to the answer. I am currently stuck on an issue related to the following error.
UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 17): Error: Protocol error (Target.createTarget): Target closed.
This error occurs once upon every iteration of the while loop. Though I'm not sure what I may be doing incorrectly.
Could someone help me do the following:
1) Diagnose the source of the error.
2) Potentially find a more effective way to traverse a large list of URLs asynchronously.
async function subProc(list, batchSize) {
let subList = null;
let i = 0;
while (list.length > 0) {
let browser = await puppeteer.launch();
subList = list.splice(0, batchSize);
console.log("Master List Size :: " + list.length);
console.log("SubList Size :: " + subList.length);
for (let j = 0; j < subList.length; j++) {
promiseArray.push(new Promise((resolve, reject) => {
resolve(pageScrape(subList[j], browser));
}));
}
Promise.all(promiseArray)
.then(response => {
procArray.concat(response);
});
promiseArray = new Array();
try {
await browser.close();
} catch(ex){
console.log(ex);
}
};
}
async function pageScrape(url, browser) {
let page = await browser.newPage();
await page.goto(url, {
timeout: 0
});
await page.waitFor(1000);
return await page.evaluate(() => {
let appTitle = document.querySelector('').innerText;
let companyName = document.querySelector('').innerText;
let dateListed = document.evaluate("", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.innerText;
let category = document.evaluate("']//a//strong", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.innerText;
/* */
return {
appTitle,
companyName,
dateListed,
category
}
}).then(response => {
let urlData = {
id: subList[j],
appName: response.appTitle,
companyName: response.companyName,
dateListed: response.dateListed,
category: response.category
}
return urlData;
});
};
I figured out the solution to the problem I was having.
Every computer is limited in its processing ability, so instead of iterating through 1000 urls simultaneously you have to break it down into smaller pieces.
By using a PromiseAll, and iterating and scraping 10 urls at a time and storing these values in an array, I was able to throttle the processing required to iterate through all 1000 urls.
processBatch(subData, 10, procArray).then((processed)=>{
for(let i = 0; i < procArray.length; i++){
for(let j = 0; j < procArray[i].length; j++){
results.push(procArray[i][j]);
}
}
function processBatch(masterList, batchSize, procArray){
return Promise.all(masterList.splice(0, batchSize).map(async url =>
{
return singleScrape(url)
})).then((results) => {
if (masterList.length < batchSize) {
console.log('done');
procArray.push(results);
return procArray;
} else {
console.log('MasterList Size :: ' + masterList.length);
procArray.push(results);
return processBatch(masterList, batchSize, procArray);
}
})
}
I'm working on an app with both Firebase (web app) and the youtube API. The idea is to let users share their playlists and interact.
1) First, the user must identify himself. This part works very well (firebase part)
2) Then, the user must accept some conditions (the scopes) from the youtube API.
3) The Youtube API return the result of the request.
The issue is that youtube API recreate for every item of the array the HTML structure : HTML > HEAD > BODY. Plus, the response skips my header and display the all thing in a blank page instead of reacting like an include PHP (i know this is javascript but still.. ).
I know the solution rest in the function executerequest but i can't figurate how to do it.
The code of the request is in the middle of the body but for the purpose of my post, i did separate it.
var GoogleAuth;
var SCOPE = 'https://www.googleapis.com/auth/youtube.force-ssl';
function handleClientLoad() {
// Load the API's client and auth2 modules.
// Call the initClient function after the modules load.
gapi.load('client:auth2', initClient);
}
function initClient() {
// Retrieve the discovery document for version 3 of YouTube Data API.
// In practice, your app can retrieve one or more discovery documents.
var discoveryUrl = 'https://www.googleapis.com/discovery/v1/apis/youtube/v3/rest';
// Initialize the gapi.client object, which app uses to make API requests.
// Get API key and client ID from API Console.
// 'scope' field specifies space-delimited list of access scopes.
gapi.client.init({
'apiKey': '...',
'discoveryDocs': [discoveryUrl],
'clientId': '....',
'scope': SCOPE
}).then(function () {
GoogleAuth = gapi.auth2.getAuthInstance();
// Listen for sign-in state changes.
GoogleAuth.isSignedIn.listen(updateSigninStatus);
// Handle initial sign-in state. (Determine if user is already signed in.)
var user = GoogleAuth.currentUser.get();
setSigninStatus();
// Call handleAuthClick function when user clicks on
// "Sign In/Authorize" button.
$('#sign-in-or-out-button').click(function() {
handleAuthClick();
});
$('#revoke-access-button').click(function() {
revokeAccess();
});
});
}
function handleAuthClick() {
if (GoogleAuth.isSignedIn.get()) {
// User is authorized and has clicked 'Sign out' button.
$('#sign-in-or-out-button').html('Sign out');
$('#revoke-access-button').css('display', 'inline-block');
GoogleAuth.signOut();
} else {
// User is not signed in. Start Google auth flow.
GoogleAuth.signIn();
}
}
function revokeAccess() {
GoogleAuth.disconnect();
}
function setSigninStatus(isSignedIn) {
var user = GoogleAuth.currentUser.get();
var isAuthorized = user.hasGrantedScopes(SCOPE);
if (isAuthorized) {
$('#sign-in-or-out-button').html('Sign out');
$('#revoke-access-button').css('display', 'inline-block');
$('#auth-status').html('Connecté ' +
' Granted');
defineRequest();
console.log('connecté');
} else {
$('#roomRed').html('display', 'block');
$('#sign-in-or-out-button').html('Sign In/Authorize');
$('#revoke-access-button').css('display', 'none');
$('#auth-status').html('Déconnecté' +
' Denied');
console.log('déconnecté');
}
// This helper method displays a message on the page.
}
function updateSigninStatus(isSignedIn) {
setSigninStatus();
}
function createResource(properties) {
var resource = {};
var normalizedProps = properties;
for (var p in properties) {
var value = properties[p];
if (p && p.substr(-2, 2) == '[]') {
var adjustedName = p.replace('[]', '');
if (value) {
normalizedProps[adjustedName] = value.split(',');
}
delete normalizedProps[p];
}
}
for (var p in normalizedProps) {
// Leave properties that don't have values out of inserted resource.
if (normalizedProps.hasOwnProperty(p) && normalizedProps[p]) {
var propArray = p.split('.');
var ref = resource;
for (var pa = 0; pa < propArray.length; pa++) {
var key = propArray[pa];
if (pa == propArray.length - 1) {
ref[key] = normalizedProps[p];
} else {
ref = ref[key] = ref[key] || {};
}
}
};
}
return resource;
}
function removeEmptyParams(params) {
for (var p in params) {
if (!params[p] || params[p] == 'undefined') {
delete params[p];
}
}
return params;
}
function executeRequest(request) {
request.execute(function(response) {
console.log(response);
for(var i = 0; i< response.items.length; i++){
console.log(response.items[i].player.embedHtml);
document.write(response.items[i].player.embedHtml);
}
});
}
function buildApiRequest(requestMethod, path, params, properties) {
params = removeEmptyParams(params);
var request;
if (properties) {
var resource = createResource(properties);
request = gapi.client.request({
'body': resource,
'method': requestMethod,
'path': path,
'params': params
});
} else {
request = gapi.client.request({
'method': requestMethod,
'path': path,
'params': params
});
}
executeRequest(request);
}
/***** END BOILERPLATE CODE *****/
function defineRequest() {
// See full sample for buildApiRequest() code, which is not
// specific to a particular youtube or youtube method.
buildApiRequest('GET',
'/youtube/v3/playlists',
{
'mine': 'true',
'maxResults': '25',
'part': 'snippet,contentDetails,player',
'onBehalfOfContentOwner': '',
'onBehalfOfContentOwnerChannel': ''
});
/*
buildApiRequest('GET',
'/youtube/v3/playlistItems',
{
'playlistId': "PLsvlo6Soc2pc2ZlereiehdPRhm0eKjSxI",
'maxResults': '25',
'part': 'snippet,contentDetails'
});
*/
}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Mes vidéo </title>
<style>
</style>
</head>
<body>
<button id="sign-in-or-out-button"
style="margin-left: 25px">Sign In/Authorize</button>
<button id="revoke-access-button"
style="display: none; margin-left: 25px">Revoke access</button>
<div id="auth-status" style="display: inline; padding-left: 25px"></div><hr>
<div id="video-container"></div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script async defer src="https://apis.google.com/js/api.js"
onload="this.onload=function(){};handleClientLoad()"
onreadystatechange="if (this.readyState === 'complete') this.onload()">
</script>
</body>
</html>
Thank you by advance
I have a function loadMessages, I want it return an Observable.
loadMessages(chatId: string): Observable<Message[]> {
console.log('1');
this.autorun(() => {
const handle = this.subscribe('messages', chatId);
if (handle.ready()) {
console.log('2');
const messages = Messages.find().fetch();
return Observable.of(messages); // here return is not for this function, which is useless
}
});
console.log('3'); // I don't want this line run immediately
// I wish I can return here, but I cannot
}
How can I return to the function level?
Also, right now the order is 1 -> 3 -> 2. Is there any way to run 1 -> 2, and wait there until I get the data?
You can try something like this:
loadMessages(chatId: string): Observable<Message[]> {
console.log('1');
return Observable.create(observer => {
this.autorun(() => {
const handle = this.subscribe('messages', chatId);
if (handle.ready()) {
console.log('2');
const messages = Messages.find().fetch();
observer.next(messages)
}
});
});
}
Very simple example is here http://plnkr.co/edit/GADtB8QCTnNubtRu9SFv?p=preview
I've created a Meteor method to upload a file, it's working well but until the file is fully uploaded, I cannot move around, all subscriptions seem to wait that the upload finishes... is there a way to avoid that ?
Here is the code on the server :
Meteor.publish('product-photo', function (productId) {
return Meteor.photos.find({productId: productId}, {limit: 1});
});
Meteor.methods({
/**
* Creates an photo
* #param obj
* #return {*}
*/
createPhoto: function (obj) {
check(obj, Object);
// Filter attributes
obj = filter(obj, [
'name',
'productId',
'size',
'type',
'url'
]);
// Check user
if (!this.userId) {
throw new Meteor.Error('not-connected');
}
// Check file name
if (typeof obj.name !== 'string' || obj.name.length > 255) {
throw new Meteor.Error('invalid-file-name');
}
// Check file type
if (typeof obj.type !== 'string' || [
'image/gif',
'image/jpg',
'image/jpeg',
'image/png'
].indexOf(obj.type) === -1) {
throw new Meteor.Error('invalid-file-type');
}
// Check file url
if (typeof obj.url !== 'string' || obj.url.length < 1) {
throw new Meteor.Error('invalid-file-url');
}
// Check file size
if (typeof obj.size !== 'number' || obj.size <= 0) {
throw new Meteor.Error('invalid-file-size');
}
// Check file max size
if (obj.size > 1024 * 1024) {
throw new Meteor.Error('file-too-large');
}
// Check if product exists
if (!obj.productId || Meteor.products.find({_id: obj.productId}).count() !== 1) {
throw new Meteor.Error('product-not-found');
}
// Limit the number of photos per user
if (Meteor.photos.find({productId: obj.productId}).count() >= 3) {
throw new Meteor.Error('max-photos-reached');
}
// Resize the photo if the data is in base64
if (typeof obj.url === 'string' && obj.url.indexOf('data:') === 0) {
obj.url = resizeImage(obj.url, 400, 400);
obj.size = obj.url.length;
obj.type = 'image/png';
}
// Add info
obj.createdAt = new Date();
obj.userId = this.userId;
return Meteor.photos.insert(obj);
}
});
And the code on the client :
Template.product.events({
'change [name=photo]': function (ev) {
var self = this;
readFilesAsDataURL(ev, function (event, file) {
var photo = {
name: file.name,
productId: self._id,
size: file.size,
type: file.type,
url: event.target.result
};
Session.set('uploadingPhoto', true);
// Save the file
Meteor.call('createPhoto', photo, function (err, photoId) {
Session.set('uploadingPhoto', false);
if (err) {
displayError(err);
} else {
notify(i18n("Transfert terminé pour {{name}}", photo));
}
});
});
}
});
I finally found the solution myself.
Explication : the code I used was blocking the subscriptions because it was using only one method call to transfer all the file from the first byte to the last one, that leads to block the thread (I think, the one reserved to each users on the server) until the transfer is complete.
Solution : I splitted the file into chunks of about 8KB, and send chunk by chunk, this way the thread or whatever was blocking the subscriptions is free after each chunk transfer.
The final working solution is on that post : How to write a file from an ArrayBuffer in JS
Client Code
// data comes from file.readAsArrayBuffer();
var total = data.byteLength;
var offset = 0;
var upload = function() {
var length = 4096; // chunk size
// adjust the last chunk size
if (offset + length > total) {
length = total - offset;
}
// I am using Uint8Array to create the chunk
// because it can be passed to the Meteor.method natively
var chunk = new Uint8Array(data, offset, length);
if (offset < total) {
// Send the chunk to the server and tell it what file to append to
Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
if (!err) {
offset += length;
upload();
}
}
}
};
upload();
Server code
var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');
Meteor.methods({
uploadFileData: function(fileId, chunk) {
var fut = new Future();
var path = '/uploads/' + fileId;
// I tried that with no success
chunk = String.fromCharCode.apply(null, chunk);
// how to write the chunk that is an Uint8Array to the disk ?
fs.appendFile(path, new Buffer(chunk), function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
return fut.wait();
}
});
Improving #Karl's code:
Client
This function breaks the file into chunks and sends them to the server one by one.
function uploadFile(file) {
const reader = new FileReader();
let _offset = 0;
let _total = file.size;
return new Promise((resolve, reject) => {
function readChunk() {
var length = 10 * 1024; // chunk size
// adjust the last chunk size
if (_offset + length > _total) {
length = _total - _offset;
}
if (_offset < _total) {
const slice = file.slice(_offset, _offset + length);
reader.readAsArrayBuffer(slice);
} else {
// EOF
setProgress(100);
resolve(true);
}
}
reader.onload = function readerOnload() {
let buffer = new Uint8Array(reader.result) // convert to binary
Meteor.call('fileUpload', file.name, buffer, _offset,
(error, length) => {
if (error) {
console.log('Oops, unable to import!');
return false;
} else {
_offset += length;
readChunk();
}
}
);
};
reader.onloadend = function readerOnloadend() {
setProgress(100 * _offset / _total);
};
readChunk();
});
}
Server
The server then writes to a file when offset is zero, or appends to its end otherwise, returning a promise, as I used an asynchronous function to write/append in order to avoid blocking the client.
if (Meteor.isServer) {
var fs = require('fs');
var Future = require('fibers/future');
}
Meteor.methods({
// Upload file from client to server
fileUpload(
fileName: string,
fileData: Uint8Array,
offset: number) {
check(fileName, String);
check(fileData, Uint8Array);
check(offset, Number);
console.log(`[x] Received file ${fileName} data length: ${fileData.length}`);
if (Meteor.isServer) {
const fut = new Future();
const filePath = '/tmp/' + fileName;
const buffer = new Buffer(fileData);
const jot = offset === 0 ? fs.writeFile : fs.appendFile;
jot(filePath, buffer, 'binary', (err) => {
if (err) {
fut.throw(err);
} else {
fut.return(buffer.length);
}
});
return fut.wait();
}
}
)};
Usage
uploadFile(file)
.then(() => {
/* do your stuff */
});