Hi i am using nightmare for scrape data from website and also course details. I occur a issue :-
err: { message: 'navigation error',
code: 0,
details: 'OK',
url: 'https://www.myskills.gov.au/courses/details?Code=CHC14015' }
on each url traversal. Please suggest me to resolve this:
var Nightmare = require('nightmare')
var vo = require('vo')
var fs = require('fs')
var filesystem = require('file-system')
// var nightmare = Nightmare({show:true});
var sleep = require('sleep');
vo(run)(function(err, result) {
if (err) throw err
console.log("Hi");
})
function *run() {
var nightmare = Nightmare({show:true});
console.log("1st step *run");
yield nightmare
.goto('https://www.myskills.gov.au/courses/search/')
.wait(12000)
//.click('#select3 value="100")
.evaluate(function () {
var hrefs = [];
$('.search-result h4 a').each(function()
{
var course = new Object();
course.title = $(this).html();
course.link = $(this).attr('href');
course.code = $('.search-result .col-md-2.text-small
span').html();
hrefs.push(course);
});
return hrefs;
})
.then(function(str){
console.log("2nd step evaluate then on page load");
console.log(str);
for(var i=0; i< 8; i++)
{
console.log(i);
sleep.sleep(1);
var link = "https://www.myskills.gov.au";
var coursetitle = str[i].title
var courselink = link +str[i].link+"\n";
var coursesinglelink = link +str[i].link;
var courseData = coursetitle+"\n"+str[i].code+"\n"+courselink;
fs.appendFile('getcourselink.txt', courseData, function (err) {
if(err) console.log(err);
});
vo(run1)(function(err, result) {
if (err) console.log ("err: ",err);
console.log("Hi in run1");
//console.log(result);
});
function *run1() {
console.log("I 2nd time:-"+i);
console.log(coursesinglelink);
sleep.sleep(2);
var nightmare1 = Nightmare({show:true});
yield nightmare1
.goto(coursesinglelink)
.wait(9000)
.evaluate(function () {
var str="Hi";
var CourseDetails = $('#details #courseStructureDiv
#packagingrules').text();
str = str+"\n"+CourseDetails;
return str;
})
.then(function(str){
console.log("Run inner then",str);
fs.appendFile('getcourselink.txt', str, function (err) {
if(err) console.log(err);
});
// nightmare1.end();
});
}
//nightmare.end();
// nightmare.proc.disconnect();
// nightmare.proc.kill();
// nightmare.ended = true;
// nightmare = null;
}
});
}
Related
My app creates a new item, and I want to retrieve the key to use in a server script. The data variable returns null though. This is what I have:
function addItem(addButton) {
var addItemPage = addButton.root;
if (!addItemPage.validate()) {
return;
}
var props = addItemPage.properties;
var itemDs = addItemPage.datasource;
props.Creating = true;
itemDs.saveChanges({
success: function(key) {
props.Creating = false;
if (app.currentPage !== app.pages.EditItem) {
return;
}
var newProjectItem = itemDs.item;
newProjectItem._loadHistory();
gotoEditItemPage(newProjectItem._key, true);
return newProjectItem;
},
failure: function(error) {
props.Creating = false;
console.error(error);
}
});
gotoEditItemPage();
var data = app.datasources.ProjectItems.item._key;
google.script.run.withSuccessHandler(function(value){
alert("Created");
}).createDoco(data);
}
This is not neat by any means, but I fixed it by creating a new function:
function addItem(addButton, key) {
var addItemPage = addButton.root;
if (!addItemPage.validate()) {
return;
}
var props = addItemPage.properties;
var itemDs = addItemPage.datasource;
props.Creating = true;
itemDs.saveChanges({
success: function() {
props.Creating = false;
if (app.currentPage !== app.pages.EditItem) {
return;
}
var newProjectItem = itemDs.item;
newProjectItem._loadHistory();
gotoEditItemPage(newProjectItem._key, true);
var key = newProjectItem._key;
value(key);
},
failure: function(error) {
props.Creating = false;
console.error(error);
}
});
gotoEditItemPage();
function value(record){
var data = record;
google.script.run.withSuccessHandler(function(value){
alert("Created");
}).createDoco(data);
}
}
I'm trying to call multiple ajax in my page using $q. after all the response am storing in one array. but it seems not working correctly-
My controller-
used for loop to go over multiple pages in API and get the json.
$scope.items = [];
for (var i = 1; i < 10; i++) {
var apiURL = "https://swapi.co/api/planets?page =" + i;
searchData(apiURL).then(function(response) {
$scope.items.push(response[0].data.results);
});
}
$scope.showDetail = function(data) {
$scope.items = data.results;
$scope.items.sort(function(a, b) {
return a.population.localeCompare(b.population);
});
}
$scope.showDetail($scope.items);
$scope.highlighFont = function() {
}
My Factory-
var app = angular.module('starApp');
app.factory('searchData', function($q, $http) {
return function(apiUrl) {
var promises = [];
var deffered = $q.defer();
$http({
method : 'GET',
url : apiUrl
}).then(function(data) {
deffered.resolve(data);
}, function(error) {
deffered.reject();
})
promises.push(deffered.promise);
return $q.all(promises);
}
});
can someone correct me if am doing wrong??
You need to call $q.all() in the controller
app.factory('searchData', function($q, $http) {
return function(apiUrl) {
return $http({
method : 'GET',
url : apiUrl
});//$http returns a promise
}
});
controller:
$scope.promises = [];
for (var i = 1; i < 10; i++) {
var apiURL = "https://swapi.co/api/planets?page =" + i;
$scope.promises.push(searchData(apiURL));
}
$q.all($scope.promises).then(function(results){
console.log(results);
});
How to update a document in Document db using queries ( basically want to update a document using a stored procedure)?
The following sample might be what you need: https://github.com/aliuy/documentdb-serverside-js/blob/master/stored-procedures/update.js.
Here's a simplified version:
function updateSproc(id, update) {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
tryQueryAndUpdate();
function tryQueryAndUpdate(continuation) {
var query = {query: "select * from root r where r.id = #id", parameters: [{name: "#id", value: id}]};
var requestOptions = {continuation: continuation};
var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, function (err, documents, responseOptions) {
if (err) throw err;
if (documents.length > 0) {
tryUpdate(documents[0]);
} else {
throw new Error("Document not found.");
}
});
}
function tryUpdate(document) {
var requestOptions = {etag: document._etag};
var fields, i;
fields = Object.keys(update);
for (i = 0; i < fields.length; i++) {
document[fields[i]] = update[fields[i]];
}
var isAccepted = collection.replaceDocument(document._self, document, requestOptions, function (err, updatedDocument, responseOptions) {
if (err) throw err;
response.setBody(updatedDocument);
});
}
I am making a meteor call and getting array in return on the client side.
I want to use this array in an event inside Template but the array is not recognized outside the meteor call.
How to access this array in a Template event?
if (Meteor.isClient) {
result=[];
Meteor.call('getApiResult', function (err, result) {
if (result) {
console.log("reached meteor call")
console.log(result);
}
});
Template.dpVar.events = {
'click .addproduct' : function (err) {
for (i = 0; i < result.length; i++) {
var Temp_Name = result[i];
var Temp_Val = document.getElementById(Temp_Name).value
console.log("temp name is ", Temp_Name);
productDB.insert({ Temp_Name: Temp_Val});
console.log("temp val is ", Temp_Val);
}
}
}//Client Ends
You aren't setting result, you are overloading the name ;-)
Try this, if it doesn't work, move result = [] into the global scope.
if (Meteor.isClient) {
result = [];
Meteor.call('getApiResult', function (err, res) {
if (res) {
console.log("reached meteor call")
console.log(res);
result = res;
}
});
Template.dpVar.events = {
'click .addproduct' : function (err) {
for (i = 0; i < result.length; i++) {
var Temp_Name = result[i];
var Temp_Val = document.getElementById(Temp_Name).value
console.log("temp name is ", Temp_Name);
productDB.insert({ Temp_Name: Temp_Val});
console.log("temp val is ", Temp_Val);
}
}
}
}
getTime: function () {
host="http://www.xxxxxx.com/";
res= Meteor.bindEnvironment(function(){
var Fiber = Meteor.require('fibers');
var Future = Meteor.require('fibers/future');
var future = new Future();
request(host, function (error, response, body) {
Fiber(function(){
if (!error && response.statusCode == 200) {
$ = cheerio.load(body);
var $thumbs = $('.thumb');
for (var i = 0, l = $thumbs.length ; i < l ; i++) {
// elements
var $thumb = $($thumbs[i]);
// save info
videourl=host+$thumb.find('a').attr('href');
videothumbs = [$thumb.find('img').attr('src')];
videos=Videos.insert({title:videoTitle,thumbs:videothumbs,lastUpdated:Date.now()});
}
return videos;
}
else {
return "error";
}
}).run()
});
}) ;
return res;
}
This is a server side method and returning undefined
Without bindenvironment it is returning error to use bindenvironemnt.
Am I using it properly, can somebody tell m the modifications in my code
You need a future to return the value of inner function:
Meteor.methods({
getTime: function() {
var future = new Future();
Meteor.bindEnvironment(function() {
future.return('Some value');
});
return future.wait();
},
});
Check out this Meteorpad for an example.
If you use suggestion from #saimeunt comment, then your code could be simplified into form:
SERVER:
Meteor.methods({
getSite:function(url){
var site = HTTP.get(url);
console.log("statusCode =\n " , site.statusCode);
console.log("content.length =\n ",site.content.length);
console.log("header =\n ",site.headers);
$ = cheerio.load(content);
...
}
})
CLIENT :
Meteor.call("getSite", "http://www.google.com");
See how it works