I'm tracking requests from multiple pages across same domain. Instead of hardcoding steps I tried to automate loop which loops through array of url-s, opens it, clicks element and wait for request which needs to be evaluated.
casper.test.begin('track request', items.length, function suite(test) {
casper.start().eachThen(items,function(response){
this.thenOpen(response.data,function(response){
this.echo(response.url);
this.test.assertExist("button.track", "Button exists");
this.mouseEvent('click', 'button.track');
var res;
this.waitForResource(function check(resource){
res = resource;
return resource.url.indexOf("click_event") != -1;
}, function(){
this.echo("Resource found" + res.url);
// parse resource code
},function(){},15000);
});
casper.run(function() {
test.done();
});
});
I' ve tried with listening for resoursceRequested event, but nothing I tried gave results.
So how can I do this and then parse request to assert its value.
Thank you.
M
edit:
casper.on('resource.received', function(resource) {
var match = resource.url.match('p1=tracking');
if(match != null) this.echo(resource.url);
});
I get every request page make, but not the one I only need, the one after click on button
edit 2:
"TypeError: 'null' is not an object (evaluating 'b.parentNode')"
"[object Object]"
Related
We need to scrape VEEC Website for the total number once a week.
As an example, for the week of 17/10/2016 - 23/10/2016 the URL returns the number Total 167,356 when the search button is clicked. We want this number to be stored in our database.
I'm using coldfusion to generate the weekly dates as params and have been passing them like the above URL. But I'm unable to find a query param so that the "Search" button click event is triggered.
I've tried like this & this but nothing seems to be working.
Any pointers?
It seems like for every form submission, a CRSF token is added, which prevents malicious activity. To make matters worse for you, the CRSF token is changed for each form submission, not just for each user, which makes it virtually impossible to circumvent.
When I make a CFHTTP POST request to this form, I get HTML FileContent back, but there is no DB data within the results table cell placeholders. It seems to me that the form owner allows form submission from an HTTP request, but if the CRSF token cannot be validated, no DB data is returned.
It maybe worth asking the website owner, if there is any kind of REST API, that you can hook into...
If you want to use a headless browser PhantomJS (https://en.wikipedia.org/wiki/PhantomJS) for this, here is a script that will save the total to a text file.
At command prompt, after you install PhantomJS, run phantomjs.exe main.js.
main.js
"use strict";
var firstLoad = true;
var url = 'https://www.veet.vic.gov.au/Public/PublicRegister/Search.aspx?CreatedFrom=17%2F10%2F2016&CreatedTo=23%2F10%2F2016';
var page = require("webpage").create();
page.viewportSize = {
width: 1280,
height: 800
};
page.onCallback = function (result) {
var fs = require('fs');
fs.write('veet.txt', result, 'w');
};
page.onLoadStarted = function () {
console.log("page.onLoadStarted, firstLoad", firstLoad);
};
page.onLoadFinished = function () {
console.log("page.onLoadFinished, firstLoad", firstLoad);
if (firstLoad) {
firstLoad = false;
page.evaluate(function () {
var event = document.createEvent("MouseEvents");
event.initEvent("click", true, true);
document.querySelectorAll(".dx-vam")[3].dispatchEvent(event);
});
} else {
page.evaluate(function () {
var element = document.querySelectorAll('.dxgv')[130];
window.callPhantom(element.textContent);
});
setTimeout(function () {
page.render('veet.png');
phantom.exit();
}, 3000);
}
};
page.open(url);
The script is not perfect, you can work on it if you're interested, but as is it will save the total to a file veet.txt and also save a screenshot veet.png.
So I have this code guys
Template.mainLayout.onCreated(function () { //HERE
console.log("mainLayout created");
var context = FlowRouter.current();
// use context to access the URL state
console.log(context);
var visitedOne = context.path;
//getting the connID
var clientIp = headers.getClientIP(); // no need for this anymore
var clientConnId = Meteor.connection._lastSessionId; // HERE
console.log(clientIp);
console.log(clientConnId); //HERE
// console.log(Meteor.connection._lastSessionId);
Meteor.call("updateHistory", {clientIp,clientConnId,visitedOne}, function(error, result){
if(error){
console.log("error", error);
});
if(result){
}
});//Meteor.call
});
My problems are marked by the comments //HERE
Meteor.connection._lastSessionId returns undefined at onCreated event. However if I try to get on click event it works just fine. Why is this caused, what's a workaround for this?
You're attempting to log the session ID before the connection has received it. For example, wrap your call in a setTimeout:
...
setTimeout(() => {
console.log(Meteor.connection._lastSessionId);
}, 500);
...
You might have to tweak the timeout value a bit, but it will be logged. Using setTimeout in this fashion really isn't that reliable though, as the amount of time it takes for the session ID to get set can vary. You'll likely want to look into setting up some kind of simple polling to continuously check for the session ID, until it's set.
Basically _lastSessionId isn't yet available on the client when the template is originally created (it's probably the first template rendered in your app). However there is no need to get this on the client since you're calling a server method anyway, just use the variable directly there where it will already exist!
So simplify:
Meteor.call("updateHistory", {clientIp,clientConnId,visitedOne}, callback)
to:
Meteor.call("updateHistory", visitedOne, callback)
and get the clientIp (if necessary) and use this.connection.id on the server.
I have a page that lists a company's profile and shows its open jobs. I use iron-router to get the companies profile info, but use a Meteor.call to get active jobs once the page has been loaded. However, when I return a cursor it throws a stack size exceeded error.
organization.js
Template.organization.rendered = function() {
Meteor.call('getActiveJobs', function(error, jobs){
if(error){
console.log(error);
} else {
console.log(jobs);
}
});
}
collection
Meteor.methods({
.....
getActiveJobs: function(){
return Jobs.find({organizationId: user.profile.organizationId});
}
.....
});
this throws a "RangeError: Maximum call stack size exceeded" error.
However, I can return
return Jobs.find({organizationId: user.profile.organizationId}).fetch();
without an error, but I'm trying t return the cursor so it's easier to work with using handlebars, but I don't quite understand why I'm getting this error.
Please note, that the values returned from your method need to be transferred from server to client in the JSON format. It follows that you cannot return Objects which are not JSON-serializable (there are small exceptions here, but we can forget about them for now).
If you want to return a cursor you should use Meteor.publish instead of Meteor.methods, so
Meteor.publish('activeJobs', function () {
var user = Meteor.users.findOne({_id: this.userId});
return Jobs.find({organizationId: user.profile.organizationId});
});
Also, remember to call Meteor.subscribe('activeJobs') on the client as soon as you need this data set.
I'm trying to use a client side collection as a site configuration system. I insert documents representing my different pages, and the iron-router and navigation tabs all use them to determine what pages they are and what templates are represented by them. Each page uses a {{> contentTemplate}} inclusion helper to load it's relevant template.
It all works great, when the data has all loaded. When I restart the app on certain pages, the data hasn't loaded yet, and I receive the Exception from Deps recompute function: Error: Expected null or template in return value from inclusion function, found: undefined error.
Here's my javascript:
StoriesArray = [
{ category: 'teaching', contentTemplate: 'teachingHome', title: 'Teaching Home'},
...
];
Stories = new Meteor.Collection(null);
StoriesArray.forEach(function (story, index) {
story._id = index + '';
Stories.insert(story);
});
// in main.js
Template.teachingPost.contentTemplate = function() {
console.log(this);
console.log(this.contentTemplate);
return Template[this.contentTemplate];
};
// in router.js
this.route('teaching', {
layoutTemplate: 'teachingPost',
data: function() { return Stories.findOne({contentTemplate: 'teachingHome', category: 'teaching'}); }
});
The console logs in the contentTemplate helper above log twice, the first time as this:
Object {} main.js?1f560c50f23d9012c6b6dd54469bb32b99aa4285:45
undefined main.js?1f560c50f23d9012c6b6dd54469bb32b99aa4285:46
and the second time as this:
Object {category: "teaching", contentTemplate: "teachingHome", title: "Teaching Home"} main.js?1f560c50f23d9012c6b6dd54469bb32b99aa4285:45
teachingHome main.js?1f560c50f23d9012c6b6dd54469bb32b99aa4285:46
so the router is simply trying to load this data too early.
I've tried putting the StoriesArray loading process into different files all over my app, including lib, and even tried putting it into Meteor.startup, but it's always the same result.
The normal iron-router waitOn/subscription pattern doesn't really apply here, since this is a client side collection built with null, that has no server representation. I don't want this to have server representation, because this is static content that there's no need to go to my server for.
How do I ensure this information is done before continuing?
Untested, but per Iron Router's docs on waitOn:
Returning a subscription handle, or anything with a ready method from the waitOn function will add the handle to a wait list.
Also in general it's better to use find with data, rather than findOne, as find will return an empty cursor when the collection is empty as opposed to findOne returning undefined. So try this:
// in router.js
this.route('teaching', {
layoutTemplate: 'teachingPost',
data: function() {
return Stories.find({contentTemplate: 'teachingHome', category: 'teaching'});
},
waitOn: function() {
var handle = {};
handle.ready = function() {
if (Stories.find().count() !== 0)
return true;
else
return false;
}
return handle;
}
});
And adjust your Template.teachingPost.contentTemplate function to work with a cursor rather than an object.
I've been scratching my head as to why this code will work some of the time, but not all (or at least most of the time). I've found that it actually does run displaying the correct content in the browser some of the time, but strangely there will be days when I'll come back to the same code, run the server (as per normal) and upon loading the page will receive an error in the console: TypeError: 'undefined' is not an object (evaluating 'Session.get('x').html')
(When I receive that error there will be times where the next line in the console will read Error - referring to the err object, and other times when it will read Object - referring the data object!?).
I'm obviously missing something about Session variables in Meteor and must be misusing them? I'm hoping someone with experience can point me in the right direction.
Thanks, in advance for any help!
Here's my dummy code:
/client/del.html
<head>
<title>del</title>
</head>
<body>
{{> hello}}
</body>
<template name="hello">
Hello World!
<div class="helloButton">{{{greeting}}}</div>
</template>
My client-side javascript file is:
/client/del.js
Meteor.call('foo', 300, function(err, data) {
err ? console.log(err) : console.log(data);
Session.set('x', data);
});
Template.hello.events = {
'click div.helloButton' : function(evt) {
if ( Session.get('x').answer.toString() === evt.target.innerHTML ) {
console.log('yay!');
}
}
};
Template.hello.greeting = function() {
return Session.get('x').html;
};
And my server-side javascript is:
/server/svr.js
Meteor.methods({
doubled: function(num) {
return num * 2;
},
foo: function(lmt) {
var count = lmt,
result = {};
for ( var i = 0; i < lmt; i++ ) {
count++;
}
count = Meteor.call('doubled', count);
result.html = "<em>" + count + "</em>";
result.answer = count;
return result;
}
});
I think it's just that the session variable won't be set yet when the client first starts up. So Session.get('x') will return undefined until your method call (foo) returns, which almost certainly won't happen before the template first draws.
However after that it will be in the session, so things will probably behave right once you refresh.
The answer is to just check if it's undefined before trying to access the variable. For example:
Template.hello.greeting = function() {
if (Session.get('x')) return Session.get('x').html;
};
One of the seven principles of Meteor is:
Latency Compensation. On the client, use prefetching and model simulation to make it look like you have a zero-latency connection to the database.
Because there is latency, your client will first attempt to draw the lay-out according to the data it has at the moment your client connects. Then it will do the call and then it will update according to the call. Sometimes the call might be able to respond fast enough to be drawn at the same time.
As now there is a chance for the variable to not be set, it would throw an exception in that occasion and thus break down execution (as the functions in the call stack will not continue to run).
There are two possible solutions to this:
Check that the variable is set when using it.
return Session.get('x') ? Session.get('x').html : '';
Make sure the variable has an initial value by setting it at the top of the script.
Session.set('x', { html = '', answer = ''});
Another approach would be to add the templates once the call responds.
Meteor.call('foo', 300, function(err, data) {
Session.set('x', data);
$('#page').html(Meteor.ui.render(function() {
return Template.someName();
}));
});