Submitting form using Casperjs Error - web-scraping

I'm trying to scrape products with color and size variations.
Every variation is a form and to get the next variation I'm clicking on radio button and it is doing a form submit action on each click.
I have tried to click it using
1. “casper.getElementsInfo(element)[0].click();”
2. “document.querySelectorAll(element)[2].click();”
3. “casper.thenClick('element');”
4. “casper.evaluate(function(){document.querySelectorAll(element)[2].click();});”
After every click I'm doing a casper wait for 8 seconds !
So here is my problem Casperjs is getting error after clicking the next color/size
var casper = require('casper').create({
viewportSize: {
width: 1920,
height: 1080
},
pageSettings: {
loadImages: false,
loadPlugins: false
}
});
casper.then(function () {
self.thenOpen("https://scubapro.johnsonoutdoors.com/fins/fins/seawing-nova-fin-full-foot", function () {
casper.then(function () {
var varitoinCount = document.querySelectorAll('div#edit-attributes-field-swatch-taxonomy > div input').length;
var i = 0;
casper.repeat(varitoinCount + 1, function () {
try {
document.querySelectorAll('div#edit-attributes-field-swatch-taxonomy > div input')[i].click();
casper.log('we got to anhoter variatoin ...' + i);
} catch (error) {
console.log(error);
}
i++;
});
});
});
});
try {
casper.run();
} catch (e) {
consol.log("Error..... " + e);
}
(TypeError: undefined is not a constructor )
And btw when I'm doing the same thing in a console it works fine

You have to understand that accessing DOM is not possible in Casper environment. You should switch to browser environment to execute your DOM selectors by using Casper's evaluate method. Basically it used to get some information from within browser or to manually do some actions with javascript. To understand it you have a whole section dedicated for that in official documentation.
That being said, I amended your code to properly click on different color variation inputs.
var casper = require('casper').create({
viewportSize: {
width: 1920,
height: 1080
},
pageSettings: {
loadImages: false,
loadPlugins: false
}
});
casper
.start()
.thenOpen("https://scubapro.johnsonoutdoors.com/fins/fins/seawing-nova-fin-full-foot", function () {
var varitoinCount = this.evaluate(function() {
return document.querySelectorAll('div#edit-attributes-field-swatch-taxonomy > div input').length;
});
var i = 0;
this.repeat(varitoinCount + 1, function() {
this.evaluate(function(iterator) {
document.querySelectorAll('div#edit-attributes-field-swatch-taxonomy > div input')[i].click();
}, i);
i++;
})
});
try {
casper.run();
} catch (e) {
console.log("Error..... " + e);
}
However it still doesn't change the color since website(for some reason) needs to take 2 clicks to change the color and I suggest that you throw in some waitFor functions to be sure that color has changed.

Related

Custom Component run function before el is loaded, but before loaded event is emitted

I am writing a custom function that requires all components to be initialized (including itself).
Once all components are initialized, but before the loaded event is emitted, I want to run some custom logic over the el and then release the loaded event.
Is there a way to deal with this and is there a custom event like 'components-loaded'
I am trying to avoid the loaded event as it would interfere with other logic that requires this event while still monitoring the dom.
AFRAME.registerComponent('cool-component', {
init: function() {
this.preloadBound = this._preload.bind(this);
this.el.addEventListener('components-loaded', this.preloadBound, {once: true});
this.el.addEventListener('componentinitialized', this.preloadBound, {once: true});
this.el.addEventListener('componentchanged', this.preloadBound, {once: true});
this.el.addEventListener('componentremoved', this.preloadBound, {once: true});
this.el.addEventListener('child-attached', this.preloadBound, {once: true});
this.el.addEventListener('child-detached', this.preloadBound, {once: true});
this.preloaded = false; <-- CREATED BOOL FLAG
},
_preload: function() {
//do some work here
this.preloaded = true;
}
exposed:function(){ <-- UPDATED THIS FUNCTION
return new Promise((resolve,reject)=>{
if(!this.preloaded){
setTimeout(() => {
this.exposed().then(resolve);
}, 200);
}
else{
//do some work based on the work done in the preload
resolve()
}
});
}
});
You could use events instead of a timer - emit an event when the preloaded stuff is done:
_preload: function() {
//do some work here
emit('preloaded_done')
this.preloaded = true;
},
exposed:function(){
if (!this.preloaded) {
this.el.addEventListener('preloaded_done', this.exposed)
return;
}
// all things that are needed to be done after the preload
}
This way the exposed function will be executed after the preloaded stuff is done.
Or you could store the events in an array
init: function() {
// ... binds and stuff
this.cachedEvents = []
this.el.addEventListener('loaded', this.cacheEvents)
stuffBeforeEmittingEvents()
},
cacheEvents: function(e) {
this.cachedEvents.push(e)
this.el.removeEventListener(e.type, this.cacheEvents)
e.stopPropagation()
}
and once You do your stuff, just loop through and emit them
for(i = 0; i < this.cachedEvents.length; i++) {
this.el.emit(this.cachedEvents[i].type, this.cachedEvents[i])
}
Something like this:
init: function() {
this.events = []
this.stuff = this.stuff.bind(this);
this.cacheEvents = this.cacheEvents.bind(this);
this.emitCachedEvents = this.emitCachedEvents.bind(this);
this.el.addEventListener('loaded', this.cacheEvents)
this.stuff()
},
stuff: function() {
// whatever you need to do
this.emitCachedEvents()
},
cacheEvents(e) {
this.events.push(e)
// prevent bubbling + catching the same event
this.el.removeEventListener(e.type, this.cacheEvents)
e.stopPropagation()
},
emitCachedEvents() {
for (let i = 0; i < this.events.length; i++) {
// primitive, may require polyfill - emit(name, data)
emit(this.events[i].type, this.events[i])
}
}
Check it out in my fiddle.

Azure Media Player Silverlight fallback not working

I have used the azure media player with my project, it will play multiple adaptive bit rate streamed videos in an asp.net page, the best part is, it is working superb in html5 and flash but it will get stuck at spinner image in silverlight fallback.
Below is the code I have used.
I have also tried to get errors but it is not hitting the event listener code added for errors, but the play and pause events are working fine where flash and html5 is used, but silverlight fallback is not working at all.
<link href="https://amp.azure.net/libs/amp/1.3.0/skins/amp-default/azuremediaplayer.min.css" rel="stylesheet">
<script src="https://amp.azure.net/libs/amp/1.3.0/azuremediaplayer.min.js"></script>
<div class="marginBlock">
<h3>
<asp:Label ID="lblTitle" runat="server"><%=Title.ToString()%></asp:Label>
</h3>
<video id="<%=mediaPlayerID %>" class="azuremediaplayer amp-default-skin amp-big-play-centered">
<p class="amp-no-js">
To view this video please enable JavaScript, and consider upgrading to a web browser that supports HTML 5 video.
</p>
</video>
</div>
<p>
<asp:Label ID="lblDescription" runat="server"><%=Description.ToString()%>
</asp:Label>
</p>
<script>
$(document).ready(function () {
var playOptions = {
"nativeControlsForTouch": false,
techOrder: ['azureHtml5JS', 'flashSS', 'silverlightSS', 'html5'],
autoplay: false,
controls: true,
width: '100%',
height: '400',
logo: { enabled: false },
poster: "<%=ImageSelector%>"
}
var azurePlayer = amp('<%=mediaPlayerID%>', playOptions);
azurePlayer.src([{
src: "<%=VideoURL%>",
type: 'application/vnd.ms-sstr+xml'
}]);
azurePlayer.addEventListener("error", function () {
var errorDetails = azurePlayer.error();
var code = errorDetails.code;
var message = errorDetails.message;
alert(errorDetails + ' ' + code + " " + message);
if (azurePlayer.error().code & amp.errorCode.abortedErrStart) {
console.log("abortedErrStart");
}
else if (azurePlayer.error().code & amp.errorCode.networkErrStart) {
// MEDIA_ERR_NETWORK errors
console.log("networkErrStart");
}
else if (azurePlayer.error().code & amp.errorCode.decodeErrStart) {
// MEDIA_ERR_DECODE errors
console.log("decodeErrStart");
}
else if (azurePlayer.error().code & amp.errorCode.srcErrStart) {
// MEDIA_ERR_SRC_NOT_SUPPORTED errors
console.log("srcErrStart");
}
else if (azurePlayer.error().code & amp.errorCode.encryptErrStart) {
// MEDIA_ERR_ENCRYPTED errors
console.log("encryptErrStart");
}
else if (azurePlayer.error().code & amp.errorCode.srcPlayerMismatchStart) {
// SRC_PLAYER_MISMATCH errors
console.log("srcPlayerMismatchStart");
}
else {
// unknown errors
console.log("unknown");
}
});
azurePlayer.addEventListener('play', function () {
console.log('play');
});
azurePlayer.addEventListener('pause', function () {
console.log('pause');
});
});
Updated
Noticed I am getting following error for IE < 11.
Also I disabled the flash in firefox and removed the silverlight from techOrder, then it should hit the error event listener, it is not hitting.
This is also important for me to handle the analytics for error.
Play and Pause event listener are working fine.
Update 8/28/2015:
Fixed the JS error, it is because of multiple call to cdn of azure mentioned in link above, moved the code in master page and load it only once, browser like chrome handles duplicity of code easily but not IE.
After all the research I am lost why it is not working.
So added the following JS that will check for Silverlight and Flash and gracefully handle the error and update our analytics as well.
function getBrowserInformation() {
var ua = navigator.userAgent, tem, M = ua.match(/(opera|chrome|safari|firefox|msie|trident(?=\/))\/?\s*(\d+)/i) || [];
if (/trident/i.test(M[1])) {
tem = /\brv[ :]+(\d+)/g.exec(ua) || [];
return { name: 'IE ', version: (tem[1] || '') };
}
if (M[1] === 'Chrome') {
tem = ua.match(/\bOPR\/(\d+)/)
if (tem != null) { return { name: 'Opera', version: tem[1] }; }
}
M = M[2] ? [M[1], M[2]] : [navigator.appName, navigator.appVersion, '-?'];
if ((tem = ua.match(/version\/(\d+)/i)) != null) { M.splice(1, 1, tem[1]); }
return {
name: M[0],
version: M[1]
};};
function checkForAzureErrors() {
function isSilverlightInstalled() {
var isSilverlightInstalled = false;
try {
//check on IE
try {
var slControl = new ActiveXObject('AgControl.AgControl');
isSilverlightInstalled = true;
}
catch (e) {
//either not installed or not IE. Check Firefox/Safari
if (navigator.plugins["Silverlight Plug-In"]) {
isSilverlightInstalled = true;
}
}
}
catch (e) {
console.log(e);
}
return isSilverlightInstalled;
}
function isFlashInstalled() {
try {
return Boolean(new ActiveXObject('ShockwaveFlash.ShockwaveFlash'));
} catch (exception) {
return ('undefined' != typeof navigator.mimeTypes['application/x-shockwave-flash']);
}
}
function addErrorMessage() {
$($("#mediaPlayer.marginBlock").find('h3')).text('Media is not supported on this browser or device.');
$($("#mediaPlayer.marginBlock").find('video')).css('display', 'none');
$($("#mediaPlayer.marginBlock").find('p')).css('display', 'none');
$('.azuremediaplayer').css('display', 'none');
ga("send", "event", "Videos", "error",
getBrowserInformation().name + getBrowserInformation().version +
": is silverlight Installed " + isSilverlightInstalled() +
" and is Flash Installed " + isFlashInstalled());
}
function checkBrowser() {
if ((getBrowserInformation().name === 'MSIE' || getBrowserInformation().name === 'IE')) {
if (getBrowserInformation().version < 11) {
addErrorMessage();
}
} else if (getBrowserInformation().name === 'Firefox') {
addErrorMessage();
}
}
if ((getBrowserInformation().name === 'MSIE' || getBrowserInformation().name === 'IE')) {
if (getBrowserInformation().version < 9) { addErrorMessage() }
}
for (var key in amp.players) {
if (amp.players.hasOwnProperty(key)) {
if (isSilverlightInstalled()) {
if (!amp.players[key].isReady_) {
checkBrowser();
}
} else if (!isFlashInstalled()) {
checkBrowser();
}
}
}}
This function is called after 5 second of page load in document.ready function, giving it enough time to load and make the isReady_boolean true.
SetTimeout(function () { checkForAzureErrors(); }, 5000);
Still I am waiting for some angel to resolve this issue.
Updates: Partially Fixed
Need to add the xap refrences like older version, it will play silverlight but there is a catch, it will only work if you have one video per page.
<script>
amp.options.flashSS.swf = "http://amp.azure.net/libs/amp/1.3.0/techs/StrobeMediaPlayback.2.0.swf"
amp.options.flashSS.plugin = "http://amp.azure.net/libs/amp/1.3.0/techs/MSAdaptiveStreamingPlugin-osmf2.0.swf"
amp.options.silverlightSS.xap = "http://amp.azure.net/libs/amp/1.3.0/techs/SmoothStreamingPlayer.xap"
</script>
Fixed
As per comments from Amit Rajput
#Parshii Currently, as per the documentation, Azure Media Player doesn't support multi-instance playback. While it may work on some techs, it is not a tested scenario at this time. Please feel free to add it to the UserVoice forum (http://aka.ms/ampuservoice).
As per my testing it is working in html5 and flash but not Silverlight, for silverlight support we can try using iframes as per comments from rnrneverdies
Single Instance Media Player is working in all techs.
#Parshii Currently, as per the documentation, Azure Media Player doesn't support multi-instance playback. While it may work on some techs, it is not a tested scenario at this time. Please feel free to add it to the UserVoice forum (http://aka.ms/ampuservoice).
This may be not a complete answer, but could help you.
I made the following plugin for the Azure Media Player 1.3.0 which logs all the activity performed by the user and also the errors.
Set up it as:
var mylogFunction = function(data) { console.log(data); };
var options = {
techOrder: ["azureHtml5JS", "flashSS", "silverlightSS", "html5"],
nativeControlsForTouch: false,
loop: false,
logo: { enabled: false },
heuristicProfile: "Quick Start", //"High Quality", // could be "Quick Start"
customPlayerSettings: {
customHeuristicSettings: {
preRollInSec: 4,
windowSizeHeuristics: true
}
},
plugins: {
DebugLog: {
logFunction: mylogFunction
}
}
};
var amPlayer = amp("yourvideotagid", options);
Plugin Code:
var amp;
(function (amp) {
amp.plugin('DebugLog', DebugLog);
function DebugLog(options) {
var player = this;
var log = function (data) { console.log("Azure Media Player Log", data); }
if (options) {
if (options['logFunction']) {
log = options['logFunction'];
}
}
init();
function init() {
player.ready(handleReady);
player.addEventListener(amp.eventName.error, handleError);
}
function handleReady() {
player.addEventListener(amp.eventName.loadedmetadata, handleLoadedMetaData);
var data = {
ampVersion: "1.3.0",
appName: options['appName'],
userAgent: navigator.userAgent,
options: {
autoplay: player.options().autoplay,
heuristicProfile: player.options().heuristicProfile,
techOrder: JSON.stringify(player.options().techOrder)
}
};
logData("InstanceCreated", 1, data);
}
function handleError() {
var err = player.error();
var data = {
sessionId: player.currentSrc(),
currentTime: player.currentTime(),
code: "0x" + err.code.toString(16),
message: err.message
};
logData("Error", 0, data);
}
function handleLoadedMetaData() {
player.addEventListener(amp.eventName.playbackbitratechanged, handlePlaybackBitrateChanged);
player.addEventListener(amp.eventName.playing, handlePlaying);
player.addEventListener(amp.eventName.seeking, handleSeeking);
player.addEventListener(amp.eventName.pause, handlePaused);
player.addEventListener(amp.eventName.waiting, handleWaiting);
player.addEventListener(amp.eventName.ended, handleEnded);
if (player.audioBufferData()) {
player.audioBufferData().addEventListener(amp.bufferDataEventName.downloadfailed, function () {
var data = {
sessionId: player.currentSrc(),
currentTime: player.currentTime(),
bufferLevel: player.audioBufferData().bufferLevel,
url: player.audioBufferData().downloadFailed.mediaDownload.url,
code: "0x" + player.audioBufferData().downloadFailed.code.toString(16),
message: player.audioBufferData().downloadFailed
};
logData("DownloadFailed", 0, data);
});
}
if (player.videoBufferData()) {
player.videoBufferData().addEventListener(amp.bufferDataEventName.downloadfailed, function () {
var data = {
sessionId: player.currentSrc(),
currentTime: player.currentTime(),
bufferLevel: player.videoBufferData().bufferLevel,
url: player.videoBufferData().downloadFailed.mediaDownload.url,
code: "0x" + player.videoBufferData().downloadFailed.code.toString(16),
message: player.videoBufferData().downloadFailed
};
logData("DownloadFailed", 0, data);
});
}
var data = {
sessionId: player.currentSrc(),
isLive: player.isLive(),
duration: player.duration(),
tech: player.currentTechName(),
protection: ((player.currentProtectionInfo() && player.currentProtectionInfo()[0]) ? player.currentProtectionInfo()[0].type : "clear")
};
logData("PresentationInfo", 1, data);
}
function handlePlaybackBitrateChanged(event) {
logData("BitrateChanged", 1, eventData(event));
}
function handleWaiting(event) {
logData("Waiting", 0, eventData(event));
}
function handlePlaying(event) {
logData("Playing", 1, eventData(event));
}
function handleSeeking(event) {
logData("Seeking", 1, eventData(event));
}
function handlePaused(event) {
logData("Paused", 1, eventData(event));
}
function handleEnded(event) {
logData("Ended", 1, eventData(event));
}
function logData(eventId, level, data) {
var eventLog = {
eventId: eventId,
level: level,
data: data
};
log(eventLog);
}
function eventData(event) {
return {
sessionId: player.currentSrc(),
currentTime: player.currentTime(),
isLive: player.isLive(),
event: event.type,
presentationTimeInSec: event.presentationTimeInSec,
message: event.message ? event.message : ""
};
}
}
})(amp || (amp = {}));

Unable to iterate thru list in autocomplete using up and down arrow key in dojo

Need help..Unable to iterate thru auto suggestions using up and down arrow keys on keyboard here is little code snippet
dojo.require("dojo.NodeList-manipulate");
dojo.require("dojo.NodeList-traverse");
dojo.ready(function () {
var div = dojo.query("#list-of-items");
console.log(dojo.byId("search").getBoundingClientRect());
dojo.connect(dojo.byId("search"), "onkeyup", function (evt) {
if (dojo.byId("search").value.trim() === "") {
dojo.forEach(div.query("li"), function (elm, i) {
dojo.style(elm, {
"display": "block"
});
});
dojo.style(dojo.query("#list-of-items")[0], {
"display": "none"
});
if(evt.keyCode == 40){
return;
}else if(evt.keyCode == 38){
return;
}
} else {
dojo.style(dojo.query("#list-of-items")[0], {
"display": "inline-block"
});
}
searchTable(this.value, evt);
});
function searchTable(inputVal, e) {
console.log(inputVal);
var list = dojo.query('#list-of-items');
dojo.forEach(list.query('li'), function (elm, i) {
var found = false;
var regExp = new RegExp(inputVal, 'i');
if (regExp.test(elm.innerText)) {
found = true;
if(i===0){
dojo.attr(elm, { className: "hlight" });
}
dojo.style(elm, {
"display": "block"
});
return false;
}
if (found == true) {
dojo.style(elm, {
"display": "block"
});
} else {
dojo.style(elm, {
"display": "none"
});
}
});
}
});
and also highlight auto suggest using this css class
.hlight{
background:#faae00;
font-weight:bold;
color:#fff;
}
Please see working Fiddle here
Thanks
The best thing to do is to keep an index that contains the highlighted value, then increment/decrease that index every time the up/down arrow is pressed.
You will also have to send that index with your searchTable() function so that it can add the .hlight class to the correct elements.
The hardest part is to correct that index when someone uses the up arrow when you're already on the first element (or the down arrow when you're on the last arrow). I solved that by adding a class .visible to the elements that are visible (in stead of just adding display: block or display: none), this way you can easily query all items that are visible.
I rewrote your code a bit, ending up with this. But still, my original question is still left, why don't you use the dijit/form/ComboBox or dijit/form/FilteringSelect? Dojo already has widgets that do this for you, you don't have to reinvent the wheel here (because it probably won't be as good).

Switch to iframe with phantom.js

I would like to switch to an iframe using pure phantom.js code
Here is my first attempt
var page = new WebPage();
var url = 'http://www.theurltofectch'
page.open(url, function (status) {
if ('success' !== status) {
console.log("Error");
} else {
page.switchToFrame("thenameoftheiframe");
console.log(page.content);
phantom.exit();
}
});
It produces only the source code of the main page. Any idea ?
Notice that the iframe domain is different from the main page domain.
Please give this a try I believe it may be an async issues meaning the iframe is not present when trying to access it. I received the below snippet from another post.
var page = require('webpage').create(),
testindex = 0,
loadInProgress = false;
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
/*
page.onNavigationRequested = function(url, type, willNavigate, main) {
console.log('Trying to navigate to: ' + url);
console.log('Caused by: ' + type);
console.log('Will actually navigate: ' + willNavigate);
console.log('Sent from the page\'s main frame: ' + main);
};
*/
/*
The steps array represents a finite set of steps in order to perform the unit test
*/
var steps = [
function() {
//Load Login Page
page.open("https://www.yourpage.com");
},
function() {
//access your iframe here
page.evaluate(function() {
});
},
function() {
//any other step you want
page.evaluate(function() {
});
},
function() {
// Output content of page to stdout after form has been submitted
page.evaluate(function() {
//console.log(document.querySelectorAll('html')[0].outerHTML);
});
//render a test image to see if login passed
page.render('test.png');
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] === "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] !== "function") {
console.log("test complete!");
phantom.exit();
}
}, 50);
replace
console.log(page.content);
with
console.log(page.frameContent);
Should return the contents of the frame phantomjs switched to.
If the iframe is from another domain you may need to add the --web-security=no option like this:
phantomjs --web-security=no myscript.js
As an additional information, what xMythicx said could be true. Some iframes are rendered via Javascript after page finishes loading. If the iframe contents are empty, then you will need to wait for all resources to finish loading, before you start grabbing stuff from the page. But this is another issue, if you need an answer on this, I suggest you ask a new question about it, and I will answer there.
Had the same problem for iframes and
phantomjs --web-security=no
helped in my case :]

Conditionally load jquery throbber plugin

I have the code below that shows a throbber and makes a getJSON call to an MVC action when the user changes an entry in a select. This all works great except there is a default -- select -- element in the list for which I don't want the getJSON to run.
However, I can't work out how to apply conditional logic to hooking this event. The conditional logic is shown as the if(selectedValue == -1) But the throbber still runs as I've hooked it in the first line. I've tried removing the first line that hooks the change event and use $.throbberShow(..) inline just prior to the getJSON call but for some reason this doesn't show the throbber.
Any help greatly appreciated.
$("#selectlist").throbber("change", { ajax: false, image: "images/ajax-loader-line.gif" });
$("#selectlist").change(
function () {
var selectedValue = $("#selectlist").val();
if (selectedValue != -1) {
//Tried doing $.throbberShow(...) here without success
$.getJSON("/Candidate/GetAddress", { id: selectedValue }, function (data, textStatus) {
if (textStatus == "success") {
$("#selectlist").val(data.Line1)
$("#selectlist").val(data.Line2)
$("#selectlist").val(data.Line3)
$("#selectlist").val(data.Town)
}
$.throbberHide();
});
}
}
);
It is more a hack than a solution as throbber doesn't support conditions but this should work:
$("#selectlist").throbber("change", { ajax: false, image: "images/ajax-loader-line.gif", delay: "500" });
$("#selectlist").change(
function () {
var selectedValue = $("#selectlist").val();
if (selectedValue != -1) {
$.getJSON("/Candidate/GetAddress", { id: selectedValue }, function (data, textStatus) {
if (textStatus == "success") {
$("#selectlist").val(data.Line1)
$("#selectlist").val(data.Line2)
$("#selectlist").val(data.Line3)
$("#selectlist").val(data.Town)
}
$.throbberHide();
});
} else {
$.throbberHide();
}
}
);

Resources