Idhttp + Download + ASP.NET - asp.net

Question

I need to program an application with Delphi that goes into this site and uses the form to get an .dat file.
http://www.bmfbovespa.com.br/BancoTitulosBTC/PosicoesEmAberto.aspx?Idioma=pt-br
Via browser I just click on the second "buscar" and get the file automatically.
This is what I got so far:
program Project1;
{$APPTYPE CONSOLE}
uses
SysUtils, idHTTP, Classes, MSHTML, SHDocVw, Httpapp, System.Variants, idURI;
var
http: TIdHttp;
url: string;
code: integer;
parameters: TStringList;
Response: TStringStream;
Resultado: string;
MS: TMemoryStream;
begin
url := 'http://www.bmfbovespa.com.br/BancoTitulosBTC/PosicoesEmAberto.aspx?Idioma=pt-br';
http := TIdHTTP.Create(nil);
parameters := TStringList.Create;
Response := TStringStream.Create;
MS := TMemoryStream.Create;
try
try
Parameters.Add('__EVENTTARGET=');
Parameters.Add('__EVENTARGUMENT=');
Parameters.Add('__VIEWSTATE=/wEPDwUKMTI5NDMyNjQ4NA8WAh4QdGlwb1Bvc2ljYW9BdGl2bwspc1RpcG9Qb3NpY2Fv'+
'QXRpdm8sIEJvdmVzcGEuU2l0ZUJtZkJvdmVzcGEuQmFuY29UaXR1bG9zQlRDLldlYiwgVmVyc2lvbj0x'+
'LjAuMC4wLCBDdWx0dXJlPW5ldXRyYWwsIFB1YmxpY0tleVRva2VuPW51bGwAFgJmDw8WCB4Ebm9kZQUk'+
'YjViNDhlNjYtZTMwMC00NTMzLTgwYzktMzI2NmM5ZDY2ODBiHgpQYXJlbnROb2RlBUsvcHQtYnIvc2Vy'+
'dmljb3MvZW1wcmVzdGltby1kZS1hdGl2b3MvZW1wcmVzdGltby1kZS1hdGl2b3MuYXNweD9JZGlvbWE9'+
'cHQtYnIeC0N1cnJlbnROb2RlBTMvYmFuY290aXR1bG9zYnRjL3Bvc2ljb2VzZW1hYmVydG8uYXNweD9'+
'JZGlvbWE9cHQtYnIeCkN1cnJlbnRVcmwFMy9CYW5jb1RpdHVsb3NCVEMvUG9zaWNvZXNFbUFiZXJ0by5'+
'hc3B4P0lkaW9tYT1wdC1icmQWAgIDD2QWAgIBD2QWCgIBD2QWCAIDDxYCHgdWaXNpYmxlaGQCDA8WBh4'+
'Hb25jbGljawUQdGhpcy52YWx1ZSA9ICcnOx4Gb25ibHVyBTFpZih0aGlzLnZhbHVlID09ICcnKSB7dGh'+
'pcy52YWx1ZT0gbm9tZVRleHRCdXNjYTt9HgpvbmtleXByZXNzBSRyZXR1cm4ga2V5UHJlc3NQZXNxdWl'+
'zYSh0aGlzLGV2ZW50KTtkAg0PDxYCHg1PbkNsaWVudENsaWNrBRxyZXR1cm4gVmVyaWZpY2FyQ2FtcG9'+
'CdXNjYSgpZGQCDg8WAh4EVGV4dAWXMzxkaXYgaWQ9J21lbnUnPjx1bCBpZD0nbWVudUhvcml6Jz48bGk'+
'gaWQ9J2FibWZib3Zlc3BhJz48YSBocmVmPScjJyBjbGFzcz0nYWJtZmJvdmVzcGEnIGlkPSdsaW5rQWJ'+
'tZic+PGltZyBzcmM9Jy9zaGFyZWQvY3NzL2ltZy90cmFuc3AuZ2lmJyAvPjwvYT48dWwgb25tb3VzZW9'+
'2ZXI9ImxpbmtBYm1mLmNsYXNzTmFtZT0nYWJtZmJvdmVzcGFob3Zlcic7IiBvbm1vdXNlb3V0PSJsaW'+
'5rQWJtZi5jbGFzc05hbWU9J2FibWZib3Zlc3BhJzsiPjxsaT48YSBocmVmPScvcHQtYnIvaW50cm9zL2'+
'ludHJvLXNvYnJlLWEtYm9sc2EuYXNweCcgdGFyZ2V0PScnPk8gcXVlIGEgQm9sc2EgZmF6PC9hPjwvbG'+
'k+PGxpPjxhIGhyZWY9Jy9wdC1ici9hLWJtZmJvdmVzcGEvdmlzaXRhcy1hLWJvbHNhL3Zpc2l0YXMtYS'+
'1ib2xzYS5hc3B4JyB0YXJnZXQ9Jyc+VmlzaXRlIGEgQm9sc2E8L2E+PC9saT48bGk+PGEgaHJlZj0nL3'+
'B0LWJyL2EtYm1mYm92ZXNwYS91bmlkYWRlcy91bmlkYWRlcy5hc3B4JyB0YXJnZXQ9Jyc+Tm9zc2FzIH'+
'VuaWRhZGVzPC9hPjwvbGk+PGxpPjxhIGhyZWY9Jy9wdC1ici9hLWJtZmJvdmVzcGEvc3VzdGVudGFiaW'+
'xpZGFkZS5hc3B4JyB0YXJnZXQ9Jyc+U3VzdGVudGFiaWxpZGFkZTwvYT48L2xpPjxsaT48YSBocmVmPS'+
'dodHRwOi8vd3d3Lmluc3RpdHV0b2JtZmJvdmVzcGEub3JnLmJyL3B0LWJyL2hvbWUuYXNwJyB0YXJnZX'+
'Q9J19ibGFuayc+SW5zdGl0dXRvIEJNJkZCT1ZFU1BBPC9hPjwvbGk+PGxpPjxhIGhyZWY9Jy9wdC1ici'+
'9hLWJtZmJvdmVzcGEvdHJhYmFsaGUtY29ub3Njby90cmFiYWxoZS1jb25vc2NvLmFzcHgnIHRhcmdldD'+
'0nJz5UcmFiYWxoZSBuYSBCb2xzYTwvYT48L2xpPjxsaT48YSBocmVmPScvc2FsYS1kZS1pbXByZW5zYS'+
'9zYWxhaW1wcmVuc2EuYXNweD9pZGlvbWE9cHQtYnInIHRhcmdldD0nJz5TYWxhIGRlIEltcHJlbnNhPC'+
'9hPjwvbGk+PC91bD48L2xpPjxsaSBpZD0nbWVyY2Fkbyc+PGEgaHJlZj0nIycgY2xhc3M9J21lcmNhZ'+
'G9zJyBpZD0nbGlua01lcmNhZG8nPjxpbWcgc3JjPScvc2hhcmVkL2Nzcy9pbWcvdHJhbnNwLmdpZicgL'+
'z48L2E+PHVsIG9ubW91c2VvdmVyPSJsaW5rTWVyY2Fkby5jbGFzc05hbWU9J21lcmNhZG9zaG92ZXInO'+
'yIgb25tb3VzZW91dD0ibGlua01lcmNhZG8uY2xhc3NOYW1lPSdtZXJjYWRvcyc7Ij48bGk+PGEgaHJlZ'+
'j0nL3B0LWJyL21lcmNhZG9zL2Fjb2VzLmFzcHgnIHRhcmdldD0nJz5Bw6fDtWVzIDwvYT48L2xpPjxsa'+
'T48YSBocmVmPScvcHQtYnIvbWVyY2Fkb3MvbWVyY2Fkb3JpYXMtZS1mdXR1cm9zLmFzcHgnIHRhcmdld'+
'D0nJz5NZXJjYWRvcmlhcyBlIEZ1dHVyb3M8L2E+PC9saT48bGk+PGEgaHJlZj0nL3B0LWJyL2ludHJvcy'+
'9pbnRyby1jYW1iaW8uYXNweCcgdGFyZ2V0PScnPkPDom1iaW88L2E+PC9saT48bGk+PGEgaHJlZj0nL3B'+
'0LWJyL2ludHJvcy9pbnRyby1hdGl2b3MuYXNweCcgdGFyZ2V0PScnPkF0aXZvczwvYT48L2xpPjxsaT'+
'48YSBocmVmPScvcHQtYnIvaW50cm9zL2ludHJvLWZ1bmRvcy5hc3B4JyB0YXJnZXQ9Jyc+RnVuZG9zIC'+
'8gRVRGczwvYT48L2xpPjxsaT48YSBocmVmPScvUmVuZGEtRml4YS9SZW5kYUZpeGEuYXNweCcgdGFyZ2'+
'V0PScnPlJlbmRhIEZpeGE8L2E+PC9saT48bGk+PGEgaHJlZj0nL3B0LWJyL2ludHJvcy9pbnRyby1vd'+
'XRyb3MtdGl0dWxvcy5hc3B4JyB0YXJnZXQ9Jyc+T3V0cm9zIFTDrXR1bG9zPC9hPjwvbGk+PC91bD48L'+
'2xpPjxsaSBpZD0nY2VudHJvZGVpbmZvcm1hY29lcyc+PGEgaHJlZj0nIycgY2xhc3M9J2NlbnRyb2Rla'+
'W5mb3JtYWNvZXMnIGlkPSdsaW5rQ2VudHJvJz48aW1nIHNyYz0nL3NoYXJlZC9jc3MvaW1nL3RyYW5zc'+
'C5naWYnIC8+PC9hPjx1bCBvbm1vdXNlb3Zlcj0ibGlua0NlbnRyby5jbGFzc05hbWU9J2NlbnRyb2Rla'+
'W5mb3JtYWNvZXNob3Zlcic7IiBvbm1vdXNlb3V0PSJsaW5rQ2VudHJvLmNsYXNzTmFtZT0nY2VudHJvZ'+
'GVpbmZvcm1hY29lcyc7Ij48bGk+PGEgaHJlZj0nL3B0LWJyL2VkdWNhY2lvbmFsL2N1cnNvcy9jdXJzb'+
'3MuYXNweCcgdGFyZ2V0PScnPkN1cnNvczwvYT48L2xpPjxsaT48YSBocmVmPScvcHQtYnIvZWR1Y2Fja'+
'W9uYWwvc2ltdWxhZG9yZXMvc2ltdWxhZG9yZXMuYXNweCcgdGFyZ2V0PScnPlNpbXVsYWRvcmVzPC9hP'+
'jwvbGk+PGxpPjxhIGhyZWY9Jy9wdC1ici9lZHVjYWNpb25hbC9vcmNhbWVudG8tcGVzc29hbC5hc3B4J'+
'yB0YXJnZXQ9Jyc+T3LDp2FtZW50byBwZXNzb2FsPC9hPjwvbGk+PGxpPjxhIGhyZWY9Jy9zaGFyZWQva'+
'WZyYW1lLmFzcHg/aWRpb21hPXB0LWJyJnVybD1odHRwOi8vd3d3LmJtZmJvdmVzcGEuY29tLmJyL3B0L'+
'WJyL2VkdWNhY2lvbmFsL2VkdWNhci9Gb3JtSW5zY3JpY2FvUGFsZXN0cmFBY2Vzc29JbnN0LmFzcCc'+
'gdGFyZ2V0PScnPlBhbGVz');
Parameters.Add('__EVENTVALIDATION=/wEWMAKU1bzjDAKatY+lDgLz2ISXCALR05XvBgKW'+
'jICHCwKn1oTUCwLzhvO8BQLzht/hDALzhsuGBALzhteEAQLzhsOpCALzhq/ODwLzhpvzBgLzhqfxAwL'+
'zhpOWCwLzhv+6AgLzhuvfCQLzhvfdBgLzhuOCDgLzhs+nBQLzhrvMDALzhsfKCQLzhrNvAvOGn5QIAv'+
'OGi7kPAvOGl7cMAvOGg9wDAvOG74ALAvOG26UCAvOG56MPAvOG08gGAvOGv+0NAo3a1W8CjNrVbwKP2'+
'tVvAo7a1W8CidrVbwKI2tVvAova1W8CitrVbwKV2tVvApTa1W8CpqKfswMCrqXL7AcChv7z9w4C4/vLv'+
'gUCo/HJ+QsCtLPk8g6HXv3ITGyMQJG6GJIiOc0sGh7cpg==');
Parameters.Add('ctl00$ucTopo$btnBusca=Busca');
Parameters.Add('ctl00$menuBOVESPASecundario=');
Parameters.Add('ctl00$contentPlaceHolderConteudo$tabPosicaoEmAberto={"State":{},"TabState":{"ctl00_contentPlaceHolderConteudo_tabPosicaoEmAberto_tabAcoes":{"Selected":true}}}');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaData$txtConsultaData=2014-07-22');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaData$txtConsultaData$dateInput=2014-07-22-00-00-00');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaData_txtConsultaData_calendar_SD=[]');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaData_txtConsultaData_calendar_AD=[[2014,5,22],[2014,7,22],[2014,7,22]]');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaEmpresa=');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaDataDownload$txtConsultaDataDownload=2014-07-02');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaDataDownload$txtConsultaDataDownload$dateInput=2014-07-02-00-00-00');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaDataDownload_txtConsultaDataDownload_calendar_SD=[]');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaDataDownload_txtConsultaDataDownload_calendar_AD=[[2014,5,22],[2014,7,22],[2014,7,22]]');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$btnBuscarArquivos=Buscar');
Parameters.Add('ctl00$contentPlaceHolderConteudo$mpgPaginas_Selected=0');
Parameters.Add('cboAgentesCorretorasNome=#');
Parameters.Add('cboAgentesCorretorasCodigo=#');
http.HandleRedirects := true;
http.Request.UserAgent := 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36';
http.AllowCookies := True;
http.Request.ContentType := 'application/x-www-form-urlencoded';
http.Post(url,Parameters,Response);
MS.SaveToFile('C:\teste.dat');
except
WriteLn(IntToStr(code));
finally
http.Free();
FreeAndNil(parameters);
FreeAndNil(response);
ReadLn;
end;
end.
I don't know how to get the file or even if I sending the right parameters.
Any suggetions?
Thanks in advance!
EDIT:
I fount out that the ViewState and the EventValidation parameters change from time to time, so now I got this:
program Project1;
{$APPTYPE CONSOLE}
uses
SysUtils, idHTTP, Classes, MSHTML, SHDocVw, Httpapp, System.Variants, idURI;
function ExtractHiddenParameter(const ParameterName: string; const Request: string): string;
const
PrefixMask = 'input type="hidden" name="%s" id="%s" value="';
Suffix = '" />';
var
Prefix: string;
PrefixLength: Integer;
PrefixPosition: Integer;
SuffixPosition: Integer;
begin
Prefix := Format(PrefixMask, [ParameterName, ParameterName]);
PrefixPosition := Pos(Prefix, Request);
if PrefixPosition = 0 then
Result := ''
else
begin
PrefixLength := Length(Prefix);
Result := Copy(Request,
PrefixPosition + PrefixLength,
1 + Length(Request) - PrefixPosition - PrefixLength);
SuffixPosition := Pos(Suffix, Result);
if SuffixPosition = 0 then
Result := ''
else
Delete(Result, SuffixPosition, 1 + Length(Result) - SuffixPosition);
end;
end;
var
http: TIdHttp;
url: string;
getRequest: string;
code: integer;
parameters: TStringList;
Response: TStringStream;
Resultado: string;
sViewState: string;
sEventValidation: string;
MS: TMemoryStream;
begin
url := 'http://www.bmfbovespa.com.br/BancoTitulosBTC/PosicoesEmAberto.aspx?Idioma=pt-br';
http := TIdHTTP.Create(nil);
parameters := TStringList.Create;
Response := TStringStream.Create;
MS := TMemoryStream.Create;
try
try
http.ProxyParams.ProxyServer := 'proxy-scl';
http.ProxyParams.ProxyPort := 3128;
http.HandleRedirects := true;
http.Request.UserAgent := 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36';
http.AllowCookies := True;
getRequest := http.Get(url);
sViewState := ExtractHiddenParameter('__VIEWSTATE', getRequest);
sEventValidation := ExtractHiddenParameter('__EVENTVALIDATION', getRequest);
Parameters.Add('__EVENTTARGET=');
Parameters.Add('__EVENTARGUMENT=');
Parameters.Add('__VIEWSTATE='+sViewState);
Parameters.Add('__EVENTVALIDATION'+sEventValidation);
Parameters.Add('ctl00$ucTopo$btnBusca=Busca');
Parameters.Add('ctl00$menuBOVESPASecundario=');
Parameters.Add('ctl00$contentPlaceHolderConteudo$tabPosicaoEmAberto={"State":{},"TabState":{"ctl00_contentPlaceHolderConteudo_tabPosicaoEmAberto_tabAcoes":{"Selected":true}}}');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaData$txtConsultaData=2014-07-22');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaData$txtConsultaData$dateInput=2014-07-22-00-00-00');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaData_txtConsultaData_calendar_SD=[]');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaData_txtConsultaData_calendar_AD=[[2014,5,22],[2014,7,22],[2014,7,22]]');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaDataDownload$txtConsultaDataDownload=2014-07-02');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$txtConsultaDataDownload$txtConsultaDataDownload$dateInput=2014-07-02-00-00-00');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaDataDownload_txtConsultaDataDownload_calendar_SD=[]');
Parameters.Add('ctl00_contentPlaceHolderConteudo_acoes_txtConsultaDataDownload_txtConsultaDataDownload_calendar_AD=[[2014,5,22],[2014,7,22],[2014,7,22]]');
Parameters.Add('ctl00$contentPlaceHolderConteudo$acoes$btnBuscarArquivos=Buscar');
Parameters.Add('ctl00$contentPlaceHolderConteudo$mpgPaginas_Selected=0');
Parameters.Add('cboAgentesCorretorasNome=#');
Parameters.Add('cboAgentesCorretorasCodigo=#');
http.Head(url);
code := http.ResponseCode;
http.Request.ContentType := 'application/x-www-form-urlencoded';
http.Post(url,Parameters,Response);
MS.SaveToFile('C:\teste.dat');
except
on E: EIdHTTPProtocolException do
code := http.ResponseCode;
end;
WriteLn(IntToStr(code));
finally
http.Free();
FreeAndNil(parameters);
FreeAndNil(response);
ReadLn;
end;
end.

Answer 1

You were very close.
just change the line
MS.SaveToFile('C:\teste.dat');
into
Response.SaveToFile('C:\teste.dat');
you can remove the TMemoryStream from your code, it is not needed