Speed up webcrawler - web-scraping

I wrote this script, which works perfectly, except it is taking about 5 minutes (give a few) to run. What would be the best way to speed this up?
Kind regards, Rob
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
int divnumber = 17476;
HtmlAgilityPack.HtmlWeb web = new HtmlAgilityPack.HtmlWeb();
while (divnumber < 18500)
{
string DivUrl = "https://www.hattrick.org/nl/World/Series/?LeagueLevelUnitID=" + divnumber;
HtmlAgilityPack.HtmlDocument doc = web.Load(DivUrl);
var ClassShy = doc.DocumentNode.SelectNodes("//a[#class='shy']");
if (ClassShy != null)
{
ClassShy.ToList();
int i = 0;
foreach (var item in ClassShy)
{
i++;
}
int divForList = divnumber - 17475;
if (i > 4)
{
Console.WriteLine("VI." + divForList + " - " + i);
}
}
divnumber++;
}
sw.Stop();
TimeSpan ts = sw.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}:{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("\nDone in " + elapsedTime);
sw.Reset();

Related

ASP.NET - Input string was not in a correct format

I am getting an error saying my input string was not in a correct format when I try to get, multiply and display I stored data's in cookies.
It says there was an error in a part in total = total + (Convert.ToInt32(a[2].ToString()) * Convert.ToInt32(a[3].ToString()));
Somebody help me please. Here is my code:
protected void Page_Init(object sender, EventArgs e)
{
DataTable dt = new DataTable();
dt.Columns.AddRange(new DataColumn[7] { new DataColumn("product_name"), new DataColumn("product_desc"), new DataColumn("product_price"), new DataColumn("product_qty"), new DataColumn("product_images"), new DataColumn("id"), new DataColumn("product_id") });
if (Request.Cookies["aa"] != null)
{
s = Convert.ToString(Request.Cookies["aa"].Value);
string[] strArr = s.Split('|');
for (int i = 0; i < strArr.Length; i++)
{
t = Convert.ToString(strArr[i].ToString());
string[] strArr1 = t.Split(',');
for (int j = 0; j < strArr1.Length; j++)
{
a[j] = strArr1[j].ToString();
}`enter code here`
dt.Rows.Add(a[0].ToString(), a[1].ToString(), a[2].ToString(), a[3].ToString(), a[4].ToString(), i.ToString(), a[5].ToString());
total = total + (Convert.ToInt32(a[2].ToString()) * Convert.ToInt32(a[3].ToString()));
totalcount = totalcount + 1;
cart_items.Text = totalcount.ToString();
cart_price.Text = total.ToString();
}
}
I recomment you to use int.TryParse(...) if you want to convert form string.
It could be like this:
int var2, var3 = 0;
if(int.TryParse(a[2].ToString(), out var2)
&& int.TryParse(a[3].ToString(), out var3))
{
total += (var2 * var3);
}

Alternative to System.Web.Security.Membership.GeneratePassword in aspnetcore (netcoreapp1.0)

Is there any alternative to System.Web.Security.Membership.GeneratePassword in AspNetCore (netcoreapp1.0).
The easiest way would be to just use a Guid.NewGuid().ToString("n") which is long enough to be worthy of a password but it's not fully random.
Here's a class/method, based on the source of Membership.GeneratePassword of that works on .NET Core:
public static class Password
{
private static readonly char[] Punctuations = "!##$%^&*()_-+=[{]};:>|./?".ToCharArray();
public static string Generate(int length, int numberOfNonAlphanumericCharacters)
{
if (length < 1 || length > 128)
{
throw new ArgumentException(nameof(length));
}
if (numberOfNonAlphanumericCharacters > length || numberOfNonAlphanumericCharacters < 0)
{
throw new ArgumentException(nameof(numberOfNonAlphanumericCharacters));
}
using (var rng = RandomNumberGenerator.Create())
{
var byteBuffer = new byte[length];
rng.GetBytes(byteBuffer);
var count = 0;
var characterBuffer = new char[length];
for (var iter = 0; iter < length; iter++)
{
var i = byteBuffer[iter] % 87;
if (i < 10)
{
characterBuffer[iter] = (char)('0' + i);
}
else if (i < 36)
{
characterBuffer[iter] = (char)('A' + i - 10);
}
else if (i < 62)
{
characterBuffer[iter] = (char)('a' + i - 36);
}
else
{
characterBuffer[iter] = Punctuations[i - 62];
count++;
}
}
if (count >= numberOfNonAlphanumericCharacters)
{
return new string(characterBuffer);
}
int j;
var rand = new Random();
for (j = 0; j < numberOfNonAlphanumericCharacters - count; j++)
{
int k;
do
{
k = rand.Next(0, length);
}
while (!char.IsLetterOrDigit(characterBuffer[k]));
characterBuffer[k] = Punctuations[rand.Next(0, Punctuations.Length)];
}
return new string(characterBuffer);
}
}
}
I've omitted the do...while loop over the CrossSiteScriptingValidation.IsDangerousString. You can add that back in yourself if you need it.
You use it like this:
var password = Password.Generate(32, 12);
Also, make sure you reference System.Security.Cryptography.Algorithms.
System.Random doesn't provide enough entropy when used for security reasons.
https://cwe.mitre.org/data/definitions/331.html
Why use the C# class System.Random at all instead of System.Security.Cryptography.RandomNumberGenerator?
Please see the example below for a more secure version of #khellang version
public static class Password
{
private static readonly char[] Punctuations = "!##$%^&*()_-+[{]}:>|/?".ToCharArray();
public static string Generate(int length, int numberOfNonAlphanumericCharacters)
{
if (length < 1 || length > 128)
{
throw new ArgumentException("length");
}
if (numberOfNonAlphanumericCharacters > length || numberOfNonAlphanumericCharacters < 0)
{
throw new ArgumentException("numberOfNonAlphanumericCharacters");
}
using (var rng = RandomNumberGenerator.Create())
{
var byteBuffer = new byte[length];
rng.GetBytes(byteBuffer);
var count = 0;
var characterBuffer = new char[length];
for (var iter = 0; iter < length; iter++)
{
var i = byteBuffer[iter] % 87;
if (i < 10)
{
characterBuffer[iter] = (char)('0' + i);
}
else if (i < 36)
{
characterBuffer[iter] = (char)('A' + i - 10);
}
else if (i < 62)
{
characterBuffer[iter] = (char)('a' + i - 36);
}
else
{
characterBuffer[iter] = Punctuations[GetRandomInt(rng, Punctuations.Length)];
count++;
}
}
if (count >= numberOfNonAlphanumericCharacters)
{
return new string(characterBuffer);
}
int j;
for (j = 0; j < numberOfNonAlphanumericCharacters - count; j++)
{
int k;
do
{
k = GetRandomInt(rng, length);
}
while (!char.IsLetterOrDigit(characterBuffer[k]));
characterBuffer[k] = Punctuations[GetRandomInt(rng, Punctuations.Length)];
}
return new string(characterBuffer);
}
}
private static int GetRandomInt(RandomNumberGenerator randomGenerator)
{
var buffer = new byte[4];
randomGenerator.GetBytes(buffer);
return BitConverter.ToInt32(buffer);
}
private static int GetRandomInt(RandomNumberGenerator randomGenerator, int maxInput)
{
return Math.Abs(GetRandomInt(randomGenerator) % maxInput);
}
}

Create 2 or more text files with ASP.NET

I have created a web app which creates 1 text file. Inside this text file it is created 1000 rows with the same word "TRY AGAIN". After this each 50 rows I put a random code which means in 1000 rows, 20 rows are random.
This is my code:
static Random randNum = new Random();
public static string Random(int ran)
{
string _charachters = "ABCDEFGHIJKMLNOPQRSTUVWXYZ0123456789";
char[] chars = new char[ran];
int allowedCharCount = _charachters.Length;
for (int i = 0; i < ran; i++)
{
chars[i] = _charachters[(int)((_charachters.Length) * randNum.NextDouble())];
}
return new string(chars);
}
protected void Button1_Click(object sender, EventArgs e)
{
string pathCreate = #"C:\" + TextBox3.Text + ".txt";
if (!File.Exists(pathCreate))
{
using (StreamWriter sw = File.CreateText(pathCreate))
{
for (int i = 1; i <= int.Parse(TextBox1.Text); i++)
{
sw.WriteLine("TRY AGAIN.");
}
}
}
string pathRandom = #"C:\" + TextBox3.Text + ".txt";
string[] lines = File.ReadAllLines(pathRandom);
for (int i = 0; i < lines.Length; i += int.Parse(TextBox2.Text))
{
lines[i] = lines[i].Replace("TRY AGAIN.", Random(int.Parse("7")));
}
File.WriteAllLines(pathRandom, lines);
}
Now I want to create 2 ore more text files with one click of a button. And on each text file there will be random codes (not duplicates). Any idea?
Thank You.
I found the solution. It is late in my country and my brain barely works. :P
for(int j = 1; j <= 10; j++)
{
string pathKrijo = #"C:\inetpub\wwwroot\KODET\" + j.ToString() + ".txt";
using (StreamWriter sw = File.CreateText(pathKrijo))
{
for (int i = 1; i <= 100; i++)
{
sw.WriteLine("Provo Përsëri.");
}
}
string pathKodFitues = #"C:\inetpub\wwwroot\KODET\" + j.ToString() + ".txt";
string[] lines = File.ReadAllLines(pathKodFitues);
for (int i = 0; i < lines.Length; i += 10)
{
lines[i] = lines[i].Replace("Provo Përsëri.", Random(int.Parse("7")));
}
File.WriteAllLines(pathKodFitues, lines);
}

getting out of memory exception while using fileStream

I am facing an Issue while uploading file from a web page to server. It works fine for files upto 200 MB, but starts throwing out of memory exception.
Could you please help me
I have pasted the code below
private void UploadToServer(HttpPostedFile oHttpPostedFile)
{
string CalCheckSum = string.Empty;
try
{
string FileName = getFileName(oHttpPostedFile.FileName.Trim());
if (File.Exists(Server.MapPath("Upload") + "\\" + System.IO.Path.GetFileName(FileName)))
{
File.Delete(Server.MapPath("Upload") + "\\" + System.IO.Path.GetFileName(FileName));
}
string serverFilePath = Server.MapPath("Upload") + "\\" + System.IO.Path.GetFileName(FileName);
FileStream fs = new FileStream(serverFilePath, FileMode.CreateNew);
string strFileFormName = serverFilePath;
// Uri oUri = new Uri(strUrl);
// DFB: Upload goes into stream
Stream myStream = oHttpPostedFile.InputStream;
string _name = oHttpPostedFile.FileName;
string _contentType = oHttpPostedFile.ContentType;
// DFB: Create buffer for stream
Byte[] myBuffer;
myBuffer = new byte[10240];
if (myStream.Length == 0)
{
//Zero Bytes file Can not be processed
CalCheckSum = string.Empty;
return;
}
else if (myStream.Length > 10240)
myBuffer = new byte[10240];
else
myBuffer = new byte[myStream.Length];
StringBuilder filecontent = new StringBuilder();
int fileLength = (int)myStream.Length;
int length = (int)myStream.Length / myBuffer.Length + 1;
int lastPacketLength = (int)myStream.Length % 10240;
int count = 1;
while (myStream.Read(myBuffer, 0, myBuffer.Length) > 0)
{
if (count == length)
fs.Write(myBuffer, 0, lastPacketLength);
else
fs.Write(myBuffer, 0, myBuffer.Length);
count++;
}
fs.Close();
fs.Dispose();
myStream.Dispose();
myBuffer = null;
myStream = null;
FileStream fileStream = File.OpenRead(serverFilePath);
byte[] pbytCombinedArrays = new byte[fileLength];
int numBytesToRead = fileLength;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fileStream.Read(pbytCombinedArrays, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
break;
numBytesRead += n;
numBytesToRead -= n;
}
fileStream.Dispose();
fileStream.Close();
}
You get the exception because you are trying to allocate more memory than is allowed for the application. Web applications are generally limited to about 300 MB.
The solution would be to avoid reading the entire file into memory. It's simply too large for a web application to handle all at once.

How to get current frame of currently playing video file?

So we have flv file, we play it with mx:vidodisplay for example. how to get on which stream frame we are currently on?
you can check the nearest keyframe to the current time in stream metadata
upd
when creating a stream you need to handle its' onMetaData call:
private var metaInfo: Object;
private function initStream():void{
stream = new NetStream(conn);
stream.bufferTime = 5;
stream.addEventListener(NetStatusEvent.NET_STATUS, onStatus);
stream.client = new Object();
stream.client.onMetaData = onMetaData;/*this is what you need*/
video.attachNetStream(stream);
}
private function onMetaData(info:Object):void {
metaInfo = info;
var tmpstr:String = '';
for(var s:String in info){
var tstr:String = s + ' = ' + info[s] + '\n';
tmpstr += tstr.indexOf('object') == -1 ? tstr : '';
for(var a:String in info[s]){
var ttstr:String = s + ':' + a + ' = ' + info[s][a] + '\n';
tmpstr += ttstr.indexOf('object') == -1 ? ttstr : '';
for(var c:String in info[s][a]){
var tttstr:String = s + ':' + a + ':' + c + ' = ' + info[s][a][c] + '\n';
tmpstr += tttstr.indexOf('object') == -1 ? tttstr : '';
}
}
}
trace(tmpstr);
}
in this trace you'll see if the streams' metadata has items like:
seekpoints:93:offset = 10342550
seekpoints:93:time = 165.799
or maybe:
keyframes:times = 0,0.48,0.96,1.44,1.92,2.4,2.88,3.36,3.84,4.32,4.8,5.28,5.76,6.24
keyframes:filepositions = 1063,95174,136998,176043,209542,239148,271062,302006,331724,363948,395039,427503,456317,483313
it depends on encoder settings, if your metadata has any object of this kind (metadata['keyframes'], metadata['seekpoints'] etc) you can do the following:
for (var i:int = 0; i < metaInfo['keyframes']['times'].length; i++) {
if (stream.time < metaInfo['keyframes']['times'][i]) {
var keyFrameNum: int = (metaInfo['keyframes']['times'][i] - stream.time < stream.time - metaInfo['keyframes']['times'][i - 1]) ? i : i - 1;
}
}
I did a static class to parse netstream metadata object to as3 object. You can use JSON.stringify(parse(info)) to check all attribute in info. This class just draftly implement. May be some bugs inside.
public class NetStreamMetaData
{
public static function parse(object:Object, isArray:Boolean = false):Object{
var ret:Object = {};
if(isArray)
ret = [];
var k:String;
for(k in object){
if(isNaN(Number(k))){
if(object[k] is Array){
ret[k] = parse(object[k], true);
}else{
ret[k] = object[k];
}
}else{
if(object[k] is Array){
ret.push(parse(object[k], false));
}else{
ret.push(object[k]);
}
}
}
return ret;
}
}

Resources