I'm running a little project that requires some of Amazon books's preview content (they can be either PNG images or html content).
For example, this book: https://www.amazon.com/gp/product/B00JNYEXCK/.
When clicking to the "Look inside" badge (img tag with id="sitbLogoImg"), a new frame appears, showing the preview content of this book. It has 2 version, printed preview (which are PNG images, these I can get a hold of) and kindle preview (which is iframe document).
I'm stuck with the iframe for kindle preview, which basically looks like this:
<div id="scrollElm-0" class="pageHtml">
<div id="sitbReaderKindleSample">
<iframe id="sitbReaderFrame">
<html>
<head></head>
<body>
<p>.......</p>
<div>......</div>
....
</body>
</html>
</iframe>
</div>
</div>
Here's my CasperJS script:
var fs = require('fs');
var casper = require('casper').create({
pageSettings: {
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36'
}
});
casper.options.viewportSize = {
width: 1366,
height: 768
};
casper.options.waitTimeout = 10000;
// use any cookies
var cookieFilename = "cookies.txt";
var data = fs.read(cookieFilename);
if (data) {
phantom.cookies = JSON.parse(data);
}
casper.start('https://www.amazon.com/gp/product/B00JNYEXCK/', function() {
this.echo(this.status(true));
this.captureSelector('before.png', 'html');
});
casper.waitForSelector('img#sitbLogoImg', function() {
//this.captureSelector('before.png','html');
});
casper.then(function() {
this.click('img#sitbLogoImg');
});
casper.waitForSelector('div#sitbLBHeader', function() {
});
var lis_content = '';
casper.wait(3000, function() {
this.captureSelector('after.png', 'html');
});
casper.withFrame(1, function() {
lis_content = this.getHTML();
this.captureSelector('lis_content.png', 'html');
});
//Write the sitbReaderFrame to file
casper.then(function() {
var lis_content_filename = 'lis_content.html';
fs.write(lis_content_filename, lis_content, 644);
});
// write the cookies
casper.wait(1000, function() {
var cookies = JSON.stringify(phantom.cookies);
fs.write(cookieFilename, cookies, 644);
});
casper.run();
The problem is the iframe only has id="sitbReaderFrame" but no name, I've tried casperjs.withFrame with frame index number from 0 to 4 but it doesn't seems to exits in CapserJS view.
I would like to hear any advice from you, as I'm really stuck here. Thank you very much and sorry for my bad English.
CasperJS script:
function on_init (page){
var width='1600',height='900';
page.viewportSize = {width:width,height:height}
page.evaluate(function (width,height){
screen = {width:width,height:height,availWidth:width,availHeight:height};
innerWidth=width; innerHeight=height; outerWidth=width; outerHeight=height;
window.navigator = {
plugins: {length: 2, 'Shockwave Flash': {name: 'Shockwave Flash', filename: '/usr/lib/flashplugin-nonfree/libflashplayer.so', description: 'Shockwave Flash 11.2 r202', version: '11.2.202.440'}},
mimeTypes: {length: 2, "application/x-shockwave-flash": {description: "Shockwave Flash", suffixes: "swf", type: "application/x-shockwave-flash", enabledPlugin: {name: 'Shockwave Flash', filename: '/usr/lib/flashplugin-nonfree/libflashplayer.so', description: 'Shockwave Flash 11.2 r202', version: '11.2.202.440'}}},
appCodeName: "Mozilla",
appName: "Netscape",
appVersion: "5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.21 Safari/537.36",
cookieEnabled: 1,
languages: "en-US,en",
language: "en",
onLine: 1,
doNotTrack: null,
platform: "Linux x86_64",
product: "Gecko",
vendor: "Google Inc.",
vendorSub: "",
productSub: 20030107,
userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.21 Safari/537.36",
geolocation: {getCurrentPosition: function getCurrentPosition(){},watchPosition: function watchPosition(){},clearWatch: function clearWatch(){}},
javaEnabled: function javaEnabled(){return 0} };},width,height);};
var casper = require('casper').create({
verbose: true,
logLevel: 'debug',
waitTimeout: 5000,
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.21 Safari/537.36'
}), fs = require('fs');
casper
.on("error", function(msg){ this.echo("error: " + msg, "ERROR") })
.on("page.error", function(msg, trace){ this.echo("Page Error: " + msg, "ERROR") })
.on("remote.message", function(msg){ this.echo("Info: " + msg, "INFO") })
.on('page.initialized', on_init)
.start("https://www.amazon.com/gp/product/B00JNYEXCK/", function(){
this.click('#ebooksSitbLogoImg');
this
.capture('lis.png')
.wait(3000,function(){
var index =this.evaluate(function(){var i,x=document.querySelectorAll('iframe'),r;
for(i=0;i<x.length;i++){if(x[i].id=="sitbReaderFrame"){r=i+1}}return r;});
this
.echo("The index is: "+index,"INFO")
.capture('lis_content.png')
.withFrame(index,function(){
fs.write('lis_content.html', this.getHTML(), 644);
})
})
})
.run();
You need to use the --cookies-file option, to avoid blocking.
./casperjs --cookies-file=./cookies_1.txt casis.js >/dev/stdout
If will print:
error: CasperError: Cannot dispatch mousedown event on nonexistent selector: #ebooksSitbLogoImg
Can't avoid blocking in anyway.
In that case
Try again after reconnecting to the internet and getting new IP address.
Related
I'm attempting to scrape the first 20 pages of https://en.mzadqatar.com/qatar/cars/sale
The site has an XHR call which checks the origin page of the user before providing the next page.
If a user enters on page 2 the site will revert the user back to the home page.
The desired response can replicated by entering at the above link and then clicking the "NEXT" button at the bottom of the page.
3
If you enter directly through this link https://en.mzadqatar.com/qatar/cars/sale?page=1 the website will revert back to the home page.
I've managed to successfully move through each page with the following code using requests, however I cannot replicate the same response using scrapy.Request. Where am I going wrong.....?
Here is the successful code using the requests library:
import requests
url = "https://en.mzadqatar.com/search"
payload = "type_id=0&id=1&subCategoryId=&pagination=1&search_type=pagination&km_from=&km_to=&price_from=&price_to=&cityId=&CartypeID=&Fueltype=&subsubCategoryId=&gear=&CylinderNumber=&cars_guarantee=&car_condition=&carcolor=&manfactureYear_from=&manfactureYear_to="
headers = {
"cookie": "laravel_session=QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL",
"authority": "en.mzadqatar.com",
"accept": "*/*",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"dnt": "1",
"origin": "https://en.mzadqatar.com",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"x-requested-with": "XMLHttpRequest"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
url = "https://en.mzadqatar.com/qatar/cars/sale"
querystring = {"page":"1"}
payload = ""
headers = {
"cookie": "laravel_session=QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL",
"authority": "en.mzadqatar.com",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"dnt": "1",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}
response = requests.request("GET", url, data=payload, headers=headers, params=querystring)
print(response.text)
Here is the code using scrapy.Requests that reverts back to the homepage
import scrapy
class MzSpider(scrapy.Spider):
name = 'mz'
allowed_domains = ['mzadqatar.com']
start_urls = ['https://en.mzadqatar.com/qatar/cars/sale']
search_url = "https://en.mzadqatar.com/search"
search_body = "type_id=0&id=1&subCategoryId=&pagination=2&search_type=pagination&km_from=&km_to=&price_from=&price_to=&cityId=&CartypeID=&Fueltype=&subsubCategoryId=&gear=&CylinderNumber=&cars_guarantee=&car_condition=&carcolor=&manfactureYear_from=&manfactureYear_to="
dict_search_body = {
"type_id": 0,
"id": 1,
"subCategoryId":"",
"pagination": 1,
"search_type": "pagination",
"km_from": "",
"km_to": "",
"undefined":"" ,
"cityId": "",
"CartypeID":"" ,
"Fueltype": "",
"subsubCategoryId": "",
"gear": "",
"CylinderNumber": "",
"cars_guarantee":"",
"car_condition": "",
"carcolor": "",
"manfactureYear_from":"",
"manfactureYear_to": ""
}
search_headers = {
"authority": "en.mzadqatar.com",
"accept": "*/*",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"dnt": "1",
"origin": "https://en.mzadqatar.com",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"x-requested-with": "XMLHttpRequest"
}
url = "https://en.mzadqatar.com/qatar/cars/sale?page=1"
headers = {
"authority": "en.mzadqatar.com",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "en-US,en;q=0.9,lo;q=0.8",
"dnt": "1",
"referer": "https://en.mzadqatar.com/qatar/cars/sale",
"sec-ch-ua": "'Chromium';v='104', ' Not A;Brand';v='99', 'Google Chrome';v='104'",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "'macOS'",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sec-gpc": "1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}
cookie = {'laravel_session' : 'QYDOviHE487FjGC2FvIaAPNnNdypE9dQcupLrylL'}
def search_requests(self):
yield scrapy.Request(
url=self.search_url,
method='POST',
headers=self.search_headers,
body=self.search_body,
cookies=self.cookie,
callback=self.start_requests
)
def start_requests(self):
yield scrapy.Request(
url=self.url,
method='GET',
headers=self.headers,
body="",
cookies=self.cookie,
callback=self.parse
)
def parse(self, response):
print(response.text)
pass
Any help on how to translate requests into scrapy.Requests would be very much appreciated.
As you see you can't get DIFFERENT page results at the same time using SAME cookies. That's why you need to use DIFFERENT cookies for DIFFERENT pages (I use meta={'cookiejar': ...} for that):
import scrapy
class MzSpider(scrapy.Spider):
name = 'mz'
allowed_domains = ['mzadqatar.com']
def start_requests(self):
for page_number in range(1, 10):
payload = {
'type_id': '0',
'id': '1',
'subCategoryId': '',
'pagination': str(page_number),
'search_type': 'pagination',
'km_from': '',
'km_to': '',
'price_from': '',
'price_to': '',
'cityId': '',
'CartypeID': '',
'Fueltype': '',
'subsubCategoryId': '',
'gear': '',
'CylinderNumber': '',
'cars_guarantee': '',
'car_condition': '',
'carcolor': '',
'manfactureYear_from': '',
'manfactureYear_to': '',
}
yield scrapy.FormRequest(
url='https://en.mzadqatar.com/search',
method='POST',
formdata=payload,
headers={
'x-requested-with': 'XMLHttpRequest',
},
cb_kwargs={
'page_number': page_number,
},
meta={'cookiejar': page_number},
callback=self.get_pagination,
)
def get_pagination(self, response, page_number):
yield scrapy.Request(
url=f'https://en.mzadqatar.com/search?page={page_number}',
meta={'cookiejar': page_number},
callback=self.parse_pagination,
cb_kwargs={
'page_number': page_number,
}
)
def parse_pagination(self, response, page_number):
with open(f'Page_{page_number}.html', 'wb') as f:
f.write(response.body)
pass
I am maintaining someone else's web app (manual JavaScript files and MVC using swagger-js to connect to a .NET API).
The one POST route is returning 415 (Unsupported Media Type), but all the other GET routes work ok. I've had a look at the existing questions on this topic and they refer to incorrect specs or other issues which don't apply to me... I think.
You can quite clearly see from the Request Headers that nothing is posted in the body. But I don't know why:
POST /api/Trip/search HTTP/1.1
Accept-Encoding: gzip, deflate, br
Accept-Language: en-US,en;q=0.9
Connection: keep-alive
Content-Length: 0
Host: localhost:44393
Origin: https://localhost:44316
Referer: https://localhost:44316/
Sec-Fetch-Dest: empty
Sec-Fetch-Mode: cors
Sec-Fetch-Site: same-site
User-Agent: Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36
accept: application/json
sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="102", "Google Chrome";v="102"
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: "Windows"
The Swagger spec is generated from dotnet core 3.1 using this route:
[Route("search")]
[HttpPost]
[ProducesResponseType(StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]
public async Task<ActionResult<TripInformationSearchResponse>> TripInformationSearch(TripInformationSearchRequest request)
{
...
}
Part of the generated spec is:
"/api/Trip/search": {
"post": {
"tags": [
"Trip"
],
"operationId": "TripInformationSearch",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Cust.API.Models.Trip.TripInformationSearchRequest"
}
},
"text/json": {
"schema": {
"$ref": "#/components/schemas/Cust.API.Models.Trip.TripInformationSearchRequest"
}
},
"application/*+json": {
"schema": {
"$ref": "#/components/schemas/Cust.API.Models.Trip.TripInformationSearchRequest"
}
}
}
},
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Cust.API.Models.Trip.TripInformationSearchResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Microsoft.AspNetCore.Mvc.ProblemDetails"
}
}
}
}
}
}
}
The webapp client uses swagger-js from JavaScript (not exactly how I would do it but I'm not about to rewrite it just now) to call the search_trips function which calls the above API route:
CUSTAPI = {
swagger_client: function (urlBase) {
var specUrl = urlBase + '/swagger/v1/swagger.json';
SwaggerClient.http.withCredentials = true; // this activates CORS, if necessary
var swaggerClient = new SwaggerClient(specUrl);
return swaggerClient;
},
// ...and later
search_trips: function (keywords, tripNumbersArr, status, routeType, hasGLPostDate, successCallback) {
var failedSwaggerLoadSpecCallback = function (reason) { CUSTAPI.error_display("Unable to connect to the API: " + reason); };
var failedApiRequestCallback = function (reason) { CUSTAPI.error_display("Unable to fulfill the API request: " + reason); };
CUSTAPI.swagger_client(oldApiClient_UrlBase)
.then(
function (swaggerClient) {
var searchRequest = {
keywords: keywords,
tripNumbers: tripNumbersArr, //array
status: status,
routeType: routeType,
hasGLPostDate: hasGLPostDate
};
return swaggerClient.apis.Trip.TripInformationSearch(searchRequest); // chaining promises
}, failedSwaggerLoadSpecCallback)
.then(function (response) {
if (response.ok)
successCallback(response.obj);
//else
// return response
}, failedApiRequestCallback);
},
...
}
And from there no amount of debugging can show me why nothing is put in the body. Can you see why?
I am trying to run the notebook from node, everything is working fine except the parameters are not accepted by the notebook instead it is sending the output based on default params. I am not getting where I am doing wrong.
Below is my call:
var job_payload = {
"run_name": runName,
"existing_cluster_id": 'cluster_id',
"notebook_task":
{
"notebook_path": notebookPath
},
"notebook_params": notebook_params //{'x':1,'y':2}
}
var url = "https://<location>.<azr_databricks>.net/api/2.0/jobs/runs/submit";
var options = {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer token'
},
body: JSON.stringify(job_payload),
};
My notebook:
import json
dbutils.widgets.text("x", '3', "firstParam")
dbutils.widgets.text("y", '4', "secondParam")
x=int(dbutils.widgets.get("x"))
y=int(dbutils.widgets.get("y"))
sum=x+y
class Output:
def __init__(self, val):
self.resultTest2 = val
p1 = Output(sum)
print(p1.resultTest2)
result=json.dumps(p1.__dict__)
#RETURNING THE OUTPUT
dbutils.notebook.exit(result)
I am sending x:1 and y:2 as param but instead of getting output 3 I am getting 7 which is default value.
As I am not getting much help from the documentation, please help:
Document URL: Microsoft link
I got the answer that where I was wrong from the below link :
StackOverflow Link
the job_payload will look like below:
var job_payload = {
"run_name": runName,
"existing_cluster_id": 'cluster_id',
"notebook_task":
{
"notebook_path": notebookPath,
"base_parameters":notebook_params //{'x':1,'y':2}
},
}
var url = "https://<location>.<azr_databricks>.net/api/2.0/jobs/runs/submit";
var options = {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer token'
},
body: JSON.stringify(job_payload),
};
Now, it is working fine.
I have a camera which sends JPEG images to a webserver through a continuous multipart http stream. When I visit the IP address of the stream, the browser reads this stream as a series of images which mimics a video. I am wanting to download the files from this stream to a remote server.
I do not know how to parse the stream and the save the files to either my ubuntu server directly, or through my ruby on rails application filesystem.
Here is how the browser sees the stream:
Response Headers:
HTTP/1.1 200 OK
Content-Type: multipart/x-mixed-replace; boundary=frame
Request Headers:
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9
Cache-Control: max-age=0
Connection: keep-alive
DNT: 1
Host: my-ip-address
Please help me find the correct approach to this problem.
You can use ffmpeg to download a video stream from a continues video stream. Since you are using ubuntu, you can do it by simply running a command in your terminal and save the stream to your remote server. Following command is a sample ffmpeg command to save a live stream to your local disk.
ffmpeg.exe -y -i http://stream2.cnmns.net/hope-mp3 hopestream-latest.mp3
In above command -i indicates URL to be recorded. "hopestream-latest.mp3"
is the output mp3 file. You can replace this with your remote server file path.
I don't have a sample server which does so. I made one myself and tried to test the solution.
const request = require('request');
const fs = require('fs')
var boundary = "";
var first = null;
var last_image = "";
let next_type = 3;
let content_length = -1;
let content_type = '';
request.get({
url: "http://localhost:9192/online.png",
forever: true,
headers: {
'referer': 'http://localhost:9192/'
},
// encoding: 'utf-8'
}
)
.on('error', (err) =>
console.log(err)
).on('response', (resp) => {
// console.log(resp)
boundary = resp.headers['content-type'].split('boundary=')[1]
// 0 - data
// 1 - content-type
// 2 - content-length
// 3 - boundary
// 4 - blank line
resp.on('data', (data)=> {
switch (next_type) {
case 0:
if (data.length + last_image.length == content_length)
{
last_image = data;
next_type = 3
} else {
last_image += data;
}
break;
case 1:
if (data.toString() == "\r\n")
{
next_type = 3
} else {
content_type = data.toString().toLowerCase().split("content-type:")[1].trim()
next_type = 2
}
break;
case 2:
content_length = parseInt(data.toString().toLowerCase().split("content-length:")[1].trim())
next_type =4
break;
case 3:
// we have got a boundary
next_type = 1;
if (last_image) {
fs.writeFileSync("image.png", last_image)
}
console.log(last_image)
last_image = ""
break;
case 4:
next_type = 0;
break;
}
})
})
This is node, since you were open to non ROR solutions also. Below is the test server I had used
streamServer.js
/* Real-Time PNG-Streaming HTTP User Counter
Copyright Drew Gottlieb, 2012
Free for any use, but don't claim
that this is your work.
Doesn't work on Windows because
node-canvas only works on Linux and OSX. */
var moment = require('moment');
var http = require('http');
var _ = require('underscore');
var Backbone = require('backbone');
var Canvas = require('canvas');
var config = {
port: 9192,
host: "0.0.0.0",
updateInterval: 3000, // 5 seconds
multipartBoundary: "whyhellothere"
};
var Client = Backbone.Model.extend({
initialize: function() {
var req = this.get('req');
var res = this.get('res');
console.log("Page opened:", req.headers.referer);
res.on('close', _.bind(this.handleClose, this));
req.on('close', _.bind(this.handleClose, this));
this.sendInitialHeaders();
this.set('updateinterval', setInterval(_.bind(this.sendUpdate, this), config.updateInterval));
},
// Re-send the image in case it needs to be re-rendered
sendUpdate: function() {
if (this.get('sending')) return;
if (!this.get('imagecache')) return;
this.sendFrame(this.get('imagecache'));
},
// Sends the actual HTTP headers
sendInitialHeaders: function() {
this.set('sending', true);
var res = this.get('res');
res.writeHead(200, {
'Connection': 'Close',
'Expires': '-1',
'Last-Modified': moment().utc().format("ddd, DD MMM YYYY HH:mm:ss") + ' GMT',
'Cache-Control': 'no-store, no-cache, must-revalidate, max-age=0, post-check=0, pre-check=0, false',
'Pragma': 'no-cache',
'Content-Type': 'multipart/x-mixed-replace; boundary=--' + config.multipartBoundary
});
res.write("--" + config.multipartBoundary + "\r\n");
this.set('sending', false);
},
// Sends an image frame, followed by an empty part to flush the image through
sendFrame: function(image) {
this.set('sending', true);
this.set('imagecache', image);
var res = this.get('res');
res.write("Content-Type: image/png\r\n");
res.write("Content-Length: " + image.length + "\r\n");
res.write("\r\n");
res.write(image);
res.write("--" + config.multipartBoundary + "\r\n");
res.write("\r\n");
res.write("--" + config.multipartBoundary + "\r\n");
this.set('sending', false);
},
// Handle a disconnect
handleClose: function() {
if (this.get('closed')) return;
this.set('closed', true);
console.log("Page closed:", this.get('req').headers.referer);
this.collection.remove(this);
clearInterval(this.get('updateinterval'));
}
});
var Clients = Backbone.Collection.extend({
model: Client,
initialize: function() {
this.on("add", this.countUpdated, this);
this.on("remove", this.countUpdated, this);
},
// Handle the client count changing
countUpdated: function() {
var image = this.generateUserCountImage(this.size());
this.each(function(client) {
client.sendFrame(image);
});
console.log("Connections:", this.size());
},
// Generate a new image
generateUserCountImage: function(count) {
var canvas = new Canvas(200, 30);
var ctx = canvas.getContext('2d');
// Background
ctx.fillStyle = "rgba(100, 149, 237, 0)";
ctx.fillRect(0, 0, 200, 30);
// Text
ctx.fillStyle = "rgb(0, 100, 0)";
ctx.font = "20px Impact";
ctx.fillText("Users online: " + count, 10, 20);
return canvas.toBuffer();
}
});
function handleRequest(req, res) {
switch (req.url) {
case '/':
case '/index.html':
showDemoPage(req, res);
break;
case '/online.png':
showImage(req, res);
break;
default:
show404(req, res);
break;
}
}
function showDemoPage(req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write("<h1>Users viewing this page:</h1>");
res.write("<img src=\"/online.png\" />");
res.write("<h5>(probably won't work on IE or Opera)</h5>");
res.end();
}
function showImage(req, res) {
// If this image is not embedded in a <img> tag, don't show it.
if (!req.headers.referer) {
res.writeHead(403, {'Content-Type': 'text/html'});
res.end("You can't view this image directly.");
return;
}
// Create a new client to handle this connection
clients.add({
req: req,
res: res
});
}
function show404(req, res) {
res.writeHead(404, {'Content-Type': 'text/html'});
res.end("<h1>not found</h1><br />go home");
}
// Ready, Set, Go!
var clients = new Clients();
http.createServer(handleRequest).listen(config.port, config.host);
console.log("Started.");
PS: Taken from https://gist.github.com/dag10/48e6d25415ca92318815
So I found that ffmpeg also has featrue for saving the images
-vframes option
Output a single frame from the video into an image file:
ffmpeg -f mjpeg -i http://192.168.1.203/stream -vframes 1 out.png
This example will output one frame (-vframes 1) into a PNG file.
fps video filter
Output one image every second, named out1.png, out2.png, out3.png, etc.
ffmpeg -f mjpeg -i http://192.168.1.203/stream -vf fps=1 out%d.png
Output one image every minute, named img001.jpg, img002.jpg, img003.jpg, etc. The %03d dictates that the ordinal number of each output image will be formatted using 3 digits.
ffmpeg -f mjpeg -i http://192.168.1.203/stream -vf fps=1/60 img%03d.jpg
Output one image every ten minutes:
ffmpeg -f mjpeg -i http://192.168.1.203/stream -vf fps=1/600 thumb%04d.bmp
PS: Taken from https://trac.ffmpeg.org/wiki/Create%20a%20thumbnail%20image%20every%20X%20seconds%20of%20the%20video
I know there are many questions similar to mine out there on this subject but those don't solve my issue. I know that web services naturally parse my objects into json as part of the framework. I have manually set the request header header Accept to 'application/json, text/javascript, /; q=0.01'. I have added the <ScriptMethod(ResponseFormat:=ResponseFormat.Json)> to my web service.
It is clear that my web service is responding to the file upload request header with a text/plain response and parsing my simple FineUploaderResponse object is failing. Keep in mind that a regular jQuery AJAX call to the same web service works fine. I would prefer not to use Web API or generic handlers in place of my web as multiple websites are reliant on my framework and expect this standard.
Thanks in advance!
The code:
Public Class FineUploaderResponse
Property Success As Boolean
End Class
<WebMethod(EnableSession:=True)> _
<ScriptMethod(ResponseFormat:=ResponseFormat.Json)> _
Public Function UploadPhotos()
'next three lines are pointless, didn't help
'HttpContext.Current.Response.Clear()
'HttpContext.Current.Response.ContentType = "application/json"
'HttpContext.Current.Response.Charset = "utf-8"
Dim response As New FineUploaderResponse()
response.Success = True
Return response
End Function
Now of course this web service works if I do this:
$(document).ready(function () {
$.ajax(
{
url: "/Services/PhotosService.asmx/UploadPhotos",
type: "POST",
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
console.log(data.d);
}
});
});
The standard jquery request header is
POST http://localhost:3066/Services/PhotosService.asmx/UploadPhotos HTTP/1.1
Host: localhost:3066
Proxy-Connection: keep-alive
Content-Length: 0
Cache-Control: no-cache
Pragma: no-cache
Origin: http://localhost:3066
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31
Content-Type: application/json; charset=utf-8
Accept: application/json, text/javascript, */*; q=0.01
X-Requested-With: XMLHttpRequest
Referer: http://localhost:3066/Test.aspx
Accept-Encoding: gzip,deflate,sdch
Accept-Language: en-US,en;q=0.8
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
But I am using FineUploader and it is posting a Content-Type:multipart/form-data; Notice though the Accept header is the same for both jQuery AJAX and FineUploader requests:
POST http://localhost:3066/Services/PhotosService.asmx/UploadPhotos HTTP/1.1
Host: localhost:3066
Proxy-Connection: keep-alive
Content-Length: 110634
Cache-Control: no-cache
Pragma: no-cache
Origin: http://localhost:3066
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31
Content-Type: multipart/form-data; boundary=----WebKitFormBoundary449tHPTKEpuO5jOR
Accept: application/json, text/javascript, */*; q=0.01
X-Requested-With: XMLHttpRequest
Referer: http://localhost:3066/Sellers/photos/?pid=37344
Accept-Encoding: gzip,deflate,sdch
Accept-Language: en-US,en;q=0.8
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
The web service response from the standard Jquery ajax call is:
HTTP/1.1 200 OK
Server: ASP.NET Development Server/10.0.0.0
Date: Wed, 17 Apr 2013 00:12:21 GMT
X-AspNet-Version: 4.0.30319
Cache-Control: private, max-age=0
Content-Type: application/json; charset=utf-8
Content-Length: 22
Connection: Close
The web service response from a FineUploader post file request is:
HTTP/1.1 500 Internal Server Error
Server: ASP.NET Development Server/10.0.0.0
Date: Wed, 17 Apr 2013 00:18:26 GMT
X-AspNet-Version: 4.0.30319
Cache-Control: private
Content-Type: text/plain; charset=utf-8
Content-Length: 1936
Connection: Close
The internal server 500 error error message details are:
System.InvalidOperationException: There was an error generating the XML document. ---> System.InvalidOperationException: The type Kazork.AppCode.PhotosService+FineUploaderResponse was not expected. Use the XmlInclude or SoapInclude attribute to specify types that are not known statically.
at System.Xml.Serialization.XmlSerializationWriter.WriteTypedPrimitive(String name, String ns, Object o, Boolean xsiType)
at Microsoft.Xml.Serialization.GeneratedAssembly.XmlSerializationWriter1.Write1_Object(String n, String ns, Object o, Boolean isNullable, Boolean needType)
at Microsoft.Xml.Serialization.GeneratedAssembly.XmlSerializationWriter1.Write3_anyType(Object o)
at Microsoft.Xml.Serialization.GeneratedAssembly.ObjectSerializer1.Serialize(Object objectToSerialize, XmlSerializationWriter writer)
at System.Xml.Serialization.XmlSerializer.Serialize(XmlWriter xmlWriter, Object o, XmlSerializerNamespaces namespaces, String encodingStyle, String id)
--- End of inner exception stack trace ---
at System.Xml.Serialization.XmlSerializer.Serialize(XmlWriter xmlWriter, Object o, XmlSerializerNamespaces namespaces, String encodingStyle, String id)
at System.Xml.Serialization.XmlSerializer.Serialize(TextWriter textWriter, Object o, XmlSerializerNamespaces namespaces)
at System.Web.Services.Protocols.XmlReturnWriter.Write(HttpResponse response, Stream outputStream, Object returnValue)
at System.Web.Services.Protocols.HttpServerProtocol.WriteReturns(Object[] returnValues, Stream outputStream)
at System.Web.Services.Protocols.WebServiceHandler.WriteReturns(Object[] returnValues)
at System.Web.Services.Protocols.WebServiceHandler.Invoke()
And, in case you are curious, here is my FineUploader call:
var uploader = new qq.FineUploader({
element: document.getElementById('bootstrapped-fine-uploader'),
request: {
endpoint: "/Services/PhotosService.asmx/UploadPhotos",
forceMultipart: true,
params: { propertyId:<%=PropertyId %>},
customHeaders: { Accept: 'application/json, text/javascript, */*; q=0.01' }
},
text: {
uploadButton: '<i class="icon-upload icon-white"></i>Upload nice images.'
},
template: '<div class="qq-uploader">' +
'<pre class="qq-upload-drop-area"><span>{dragZoneText}</span></pre>' +
'<div class="qq-upload-button btn btn-success" style="width: auto;">{uploadButtonText}</div>' +
'<span class="qq-drop-processing"><span>{dropProcessingText}</span><span class="qq-drop-processing-spinner"></span></span>' +
'</div>' +
'<ul class="qq-upload-list" style="margin-top: 10px; text-align: center;"></ul>' +
'',
classes: {
success: 'alert alert-success',
fail: 'alert alert-error'
},
debug: false,
callbacks: {
// onComplete: function (id, fileName, responseJson) {
// $.when(loadThumbs()).done(function () {
// $(".qq-upload-list > .alert-success").remove();
// });
// toastr.success("Success!");
// },
onComplete: function(id, fileName, responseJSON) {
if (responseJSON.success) {
$('#file-' + id).removeClass('alert-info')
.addClass('alert-success')
.html('<i class="icon-ok"></i> ' +
'Successfully saved ' +
'“' + fileName + '”' +
'<br><img src="/images/message_ok.png" alt="' + fileName + '">');
$.when(loadThumbs()).done(function () {
$(".qq-upload-list > .alert-success").remove();
});
toastr.success("Success!");
} else {
$('#file-' + id).removeClass('alert-info')
.addClass('alert-error')
.html('<i class="icon-exclamation-sign"></i> ' +
'Error with ' +
'“' + fileName + '”: ' +
responseJSON.error);
}
},
onError: function (id, fileName, errorReason) {
toastr.error("Failed! Try again.");
}
}
});
Ok easy fix, frustrating though, web services kill a lot of my time trying to maintain a contemporary web app. But I don't have a need to migrate to Web API at this point, primarily because my web app is reliant on session (you can introduce session into Web API but that is not RESTful obviously).
So I replaced the FineUploaderResponse class with my own formatted JSON response:
<WebMethod(EnableSession:=True)> _
<ScriptMethod(ResponseFormat:=ResponseFormat.Json)> _
Public Sub UploadPhotos()
Context.Response.Write(New ResultData().GetResultDataJSON("success", "true"))
End Sub
Public Function GetResultDataJSON(key As String, value As String) As String
Dim oBuilder As StringBuilder = New StringBuilder()
oBuilder.Append("{")
oBuilder.AppendFormat("""{0}"" : {1}", key, value)
oBuilder.Append("}")
Return oBuilder.ToString()
End Function
And here is my FineUploader Javascript:
/*=================================================*/
//fineUploaderInitialize
/*=================================================*/
function createUploader() {
var uploader = new qq.FineUploader({
element: document.getElementById('bootstrapped-fine-uploader'),
request: {
endpoint: "/Services/PhotosService.asmx/UploadPhotos",
forceMultipart: true,
params: { propertyId:$('#hiddenPropertyIdUploadPhotosUserControl').val()},
customHeaders: { Accept: 'application/json, text/javascript, */*; q=0.01' },
allowedExtensions: ['gif', 'jpeg', 'jpg', 'png'],
},
text: {
uploadButton: 'Drag & drop photos into this area or CLICK HERE to upload photos'
},
template: '<div class="qq-uploader text-center" style="height:70px;background-color:white;border-radius:6px;">' +
'<div class="qq-upload-button btn btn-success col-lg-12">{uploadButtonText}</div>' +
'<span class="qq-drop-processing"><span>{dropProcessingText}</span><span class="qq-drop-processing-spinner"></span></span>' +
'<pre class="qq-upload-drop-area"><span>{dragZoneText}</span></pre>' +
'</div>' +
'<ul class="qq-upload-list" style="margin-top: 10px; text-align: center;"></ul>' +
'',
classes: {
success: 'alert alert-success',
fail: 'alert alert-error'
},
debug: false,
callbacks: {
onComplete: function(id, fileName, responseJSON) {
if (responseJSON.success) {
$('#file-' + id).removeClass('alert-info')
.addClass('alert-success')
.html('<i class="glyphicon glyphicon-ok"></i> ' +
'Successfully saved ' +
'“' + fileName + '”' +
'<br><img src="/images/message_ok.png" alt="' + fileName + '">');
$.when(loadThumbs()).done(function () {
$(".qq-upload-list > .alert-success").remove();
});
toastr.success("Success!");
} else {
$('#file-' + id).removeClass('alert-info')
.addClass('alert-error')
.html('<i class="glyphicon glyphicon-exclamation-sign"></i> ' +
'Error with ' +
'“' + fileName + '”: ' +
responseJSON.error);
}
},
onError: function (id, fileName, errorReason) {
if(errorReason == 'XHR returned response code 0'){
toastr.error('File Size Cannot Exceed 20 Megabytes');
}else{
toastr.error(errorReason);
}
}
}
});
}
/*=================================================*/