I am trying to read a specific webpage which loads in Firefox without any problems, but in Delphi I get a 404 error. How can I fix it? Maybe they don´t want bots to scrape their page...
IdHttp1 := TIdHTTP.Create(nil);
IdSSLIOHandlerSocketOpenSSL1 := TIdSSLIOHandlerSocketOpenSSL.Create(nil);
IdSSLIOHandlerSocketOpenSSL1.SSLOptions.SSLVersions := [sslvTLSv1, sslvTLSv1_1, sslvTLSv1_2];
IdHTTP1.IOHandler := IdSSLIOHandlerSocketOpenSSL1;
IdHttp1.ReadTimeout := 20000;
IdHttp1.ConnectTimeout := 3000;
IdHttp1.HandleRedirects := TRUE;
IdHttp1.Request.UserAgent := 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36';
IdHTTP1.Request.Accept := 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9';
IdHTTP1.Request.AcceptEncoding := 'gzip, deflate, br';
IdHTTP1.Request.AcceptLanguage := 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7';
IdHTTP1.AllowCookies := true;
try
HTMLText := IdHttp1.Get('https://ecagbda.r.bh.d.sendibt3.com/tr/cl/DVtXcTF4cdz4-WFm1tHD49xCh694dE4r6-n2sTyV9cXzQS_1WHzxCA3EqrKluw_X1fzriADR2oOmRFuWxqGWCfz6S_Jbbh1viVcKrbrhR6yZAkDPLl-GPE7jXp9UymHh5J2qhya1XcfAXh0l4cTIb7UXI5dFp6boutjlrL38JyiTxMGsEHQK8uVRkFtMstmMYhPrkUI8cBkiHxj3mdjVu6SXFEw6644iLwjCFZoGSuu6M95bc0fAnbLy0mDAHk2qt2ASx2u4QuKRoDIZvlTGSjPhJnUzP5n4VjPyxgu3MimDuoj2ezWmxRKIYft1PK4oP2fEx2SSJyX1-PKgAZCvfCs41ZsjgXY_Ng');
except
on E: Exception DO
begin
// ShowMessage('Exception class name = ' + E.ClassName);
// ShowMessage('Exception message = ' + E.Message);
//Halt;
Fehler := True;
end;
end;
Related
I am writing an image downloader software. But I have problem to download some images such as:
https://books.google.com/books/content?id=8_pCYmpCu6UC&pg=PT4&img=1&zoom=3&hl=en&bul=1&sig=ACfU3U10JF_yd0n7gvCllBts88O_ufTWxA&w=1280
You can check it in your browser (You should see a page with 1.Introduction title. If you could not see, it is related to your region restriction. Using US proxy would solve it).
I'm using Delphi 10.3 update 2 in Win 10. My project is 32 version.
1-I used TIdHttp to download the file but I got another image (An Image with this content: "Image not available").
procedure DownloadeImage(FURL, FFileName :string);
var
HTTPClient: TIdHTTP;
FileStream: TFileStream;
begin
try
HTTPClient := TIdHTTP.Create;
HTTPClient.HandleRedirects := True;
HTTPClient.AllowCookies := True;
//HTTPClient.IOHandler := TIdSSLIOHandlerSocketOpenSSL.Create(nil);
//(HTTPClient.IOHandler as TIdOpenSSLIOHandlerClient).Options.VerifyServerCertificate := False;
HTTPClient.Request.UserAgent := 'Mozilla/5.0'; //'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0';
HTTPClient.Request.Connection := 'keep-alive';
try
FileStream := TFileStream.Create(FFileName, fmCreate);
try
HTTPClient.Get(FURL, FileStream); //(TIdURI.URLEncode(FURL), FileStream);
finally
FileStream.Free;
end;
finally
HTTPClient.Free;
end;
except
// error handling ignored for this example
end;
end;
2-The same result when using TNetHttpClient
procedure DownloadeImage(FURL, FFileName :string);
var
HTTPClient: TNetHTTPClient;
FileStream: TFileStream;
begin
try
HTTPClient := TNetHTTPClient.Create(nil);
HTTPClient.HandleRedirects := True;
HTTPClient.AllowCookies := True;
//HTTPClient.SecureProtocols := [THTTPSecureProtocol.SSL2, THTTPSecureProtocol.SSL3, THTTPSecureProtocol.TLS1, //THTTPSecureProtocol.TLS11, THTTPSecureProtocol.TLS12];
//HTTPClient.Accept := 'image/png, image/gif, image/jpg, image/jpeg, image/tif, image/tiff, image/bmp, //image/x-bmp;q=0.9,*/*;q=0.8';
HTTPClient.UserAgent := 'Mozilla/5.0';
try
FileStream := TFileStream.Create(FFileName, fmCreate);
try
HTTPClient.Get(FURL, FileStream); //(TIdURI.URLEncode(FURL), FileStream);
finally
FileStream.Free;
end;
finally
HTTPClient.Free;
end;
except
// error handling ignored for this example
end;
end;
3-But I get the goal using the following code:
URLMon.URLDownloadToFile(nil, PChar(FURL), PChar(FFileName), 0, nil)
Why?!!!
I think it is related to some default value of TIdhttp and TNetHttpClient or about some SSL protocol.
Can anyone guide me?
I am using XE8.
My openssl dlls are in 1.0.2.13 version.
I have the next site with the documentation to get some currency exchange values:
https://api.promasters.net.br/cotacao/#documentacao
I am trying the next code, but I get an error:
'Error connecting with SSL.'#$D#$A'error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure'
if I put the next line into browser "http://api.promasters.net.br/cotacao/v1/valores" it returns the correct values.
How could I fix it?
My code:
const ss='http://api.promasters.net.br/cotacao/v1/valores';
var
s: string;
lHTTP: TIdHTTP;
HandlerSock:TIdSSLIOHandlerSocketOpenSSL;
begin
lHTTP := TIdHTTP.Create(nil);
lHTTP.Request.UserAgent := 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0';
try
HandlerSock:=TIdSSLIOHandlerSocketOpenSSL.Create(lHTTP);
HandlerSock.SSLOptions.Method := sslvSSLv23;
HandlerSock.SSLOptions.SSLVersions := [sslvTLSv1_2, sslvTLSv1_1, sslvTLSv1];
lHTTP.IOHandler := HandlerSock;
lHTTP.HandleRedirects := True;
s := lHTTP.Get(ss);
ShowMessage(s);
finally
lHTTP.Free;
end;
end;
If I use the next code, it works, but It doesn't uses Indy. I´d like pure Indy code.
const ss='https://api.promasters.net.br/cotacao/v1/valores';
var
Resp, Req: TStringStream;
recieveID: Integer;
Rio: THTTPRIO;
s, tv: string;
begin
Rio := THTTPRIO.Create(Nil);
Rio.HTTPWebNode.URL :=ss;
Rio.HTTPWebNode.CheckContentType;
Rio.HTTPWebNode.Agent :='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0';
Rio.Port := '443';
Result := '';
Resp := TStringStream.Create('');
Req := nil;
try
Req := TStringStream.Create(s, TEncoding.UTF8);
try
recieveID := Rio.HTTPWebNode.Send(Req); // Request
Rio.HTTPWebNode.Receive(recieveID, Resp, false); // Response
Resp.Position := 0;
Result:=Resp.DataString;
except
on e:exception do begin
Result := '';
end;
end;
finally
if Assigned(Req) then
Req.Free;
Resp.Free;
Rio.Free;
end;
end;
I'm try send a Get request to website. The problem is that website is recongize if the requester is a robot
const _URL = 'https://www.URL.com/';
var
sSessionID:String;
Params: TStringList;
IdSSL: TIdSSLIOHandlerSocketOpenSSL;
begin
IdSSL := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP1);
try
IdHTTP1.IOHandler := IdSSL;
IdHTTP1.AllowCookies := True;
IdHTTP1.HandleRedirects := True;
IdHTTP1.Request.UserAgent := 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0';
IdHTTP1.Request.Accept := 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
IdHTTP1.Request.AcceptLanguage := 'en-GB,en;q=0.5';
IdHTTP1.Request.Connection := 'keep-alive';
IdHTTP1.Request.ContentType := 'application/x-www-form-urlencoded';
sSessionID := IdHTTP1.Get(_URL);
{....
extracting SessionID
Params.Add('SessionID=' + 'sSessionID');
IdHTTP1.Post(_URL, Params);
.....}
finally
IdSSL.Free;
end;
The result of the IdHTTP.get is <!DOCTYPE html><head><META NAME="ROBOTS"..... Its empty i can't obtin the session ID.
The http request headers is the same what my borwser sent.
As I can have the real URL this is my best guess:
uses
Math;
...
const
_URL = 'https://www.url.com/';
var
sSessionID: string;
Params: TStringList;
IdSSL: TIdSSLIOHandlerSocketOpenSSL;
begin
IdSSL := TIdSSLIOHandlerSocketOpenSSL.Create(IdHTTP1);
try
IdHTTP1.IOHandler := IdSSL;
IdHTTP1.AllowCookies := True;
IdHTTP1.HandleRedirects := True;
IdHTTP1.Request.CustomHeaders.AddValue('X-Forwarded-For', Format('%d.%d.%d.%d', [Random(255), Random(255), Random(255), Random(255)]));
IdHTTP1.Request.UserAgent := Format('Mozilla/%d.0 (Windows NT %d.%d; rv:2.0.1) Gecko/20100101 Firefox/%d.%d.%d', [RandomRange(3, 5), RandomRange(3, 5), Random(2), RandomRange(3, 5), Random(5), Random(5)]);
IdHTTP1.Request.Accept := 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
IdHTTP1.Request.AcceptLanguage := 'en-GB,en;q=0.5';
IdHTTP1.Request.Connection := 'keep-alive';
IdHTTP1.Request.ContentType := 'application/x-www-form-urlencoded';
sSessionID := IdHTTP1.Get(_URL);
...
finally
...
end;
I am working on a program which share data to LinkedIn User’s Profile. I am using Delphi XE2, OAuth and LinkedIn API for the same. I am able to get Access token. Then I want to update status. So my code is
procedure TForm1.Button2Click(Sender: TObject);
var
IdSSLIOHandlerSocketOpenSSL1: TIdSSLIOHandlerSocketOpenSSL ;
Url,sign : String;
Response : TStringStream;
slist : TStringList;
str : WideString;
Arequest1 : TOAuthRequest;
AuthHeader : WideString;
begin
with http do
Begin
IdSSLIOHandlerSocketOpenSSL1 := TIdSSLIOHandlerSOcketOpenSSL.Create(nil);
with idSSLIOHandlerSocketOpenSSL1 do
begin
SSLOptions.Method := sslvTLSv1;
SSLOptions.SSLVersions := [sslvTLSv1];
SSLOptions.Mode := sslmBoth;
SSLOptions.VerifyMode := [];
SSLOptions.VerifyDepth := 0;
host := '';
end;
IOHandler := IdSSLIOHandlerSocketOpenSSL1;
AllowCookies := True;
Request.ContentRangeEnd := 0;
Request.ContentRangeStart := 0;
Request.ContentType := 'application/xml';
Request.ContentEncoding := 'utf-8';
Request.BasicAuthentication := False;
Request.Connection := 'Keep-Alive';
request.host := 'api.linkedin.com';
Request.Accept := 'text/xml, */*';
Request.UserAgent := 'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0' ;
HTTPOptions := [hoForceEncodeParams];
end;
Url := 'https://api.linkedin.com/v1/people/~/shares';
Consumer := nil;
Consumer := TOAuthConsumer.Create(ConsumerKey, ConsumerSecret);
Arequest1 := TOAuthRequest.Create(Url);
sign := HMAC.build_signature(Arequest1,Consumer,Token);
sign := TOAuthUtil.urlEncodeRFC3986(sign);
http.Request.CustomHeaders.Clear;
AuthHeader := 'OAuth '+ 'oauth_nonce="'+Arequest1.GenerateNonce+'",'+ 'oauth_signature_method="'+HMAC.get_name+'",'+ 'oauth_timestamp="'+Timestamp+'",'+ 'oauth_consumer_key="'+Consumer.Key+'",'+ 'oauth_token="'+Token.Key+'",'+ 'oauth_signature="'+sign+'",'+ 'oauth_version="1.0"';
http.Request.CustomHeaders.Add(AuthHeader);
slist := TStringList.Create;
slist.Text := '<?xml version="1.0" encoding="UTF-8"?><share><comment>Posting from the API using XML</comment>'+
'<content><title>A title for your share</title><submitted-url>http://developer.linkedin.com</submitted-url>'+
'<submitted-image-url>http://lnkd.in/Vjc5ec</submitted-image-url></content><visibility>'+
'<code>anyone</code></visibility></share>';
Response := TStringStream.Create;
http.Request.ContentLength := length(slist.Text);
try
http.post(Url,slist,Response);
Finally
ShowMessage(Response.DataString);
http.Free;
ARequest.Free;
end;
end;
I am facing 401 unauthorized. Please check my ques #A1rPun. Please help me
Thank you for providing a full code sample. I've seen you commented on your previous question that you use indy 10 in Delphi XE2.
The problem could be that there is something wrong with the SSL authentication within indy 10. Full explanation here What you can do is trying to get a stable version of the indy components.
The code seem to have no errors except for a couple of resources that could have been freed ;).
I really hope when updating your indy your problem will solve.
What is the easiest way to do an HTTPS POST request in Delphi? I'm not having problems with making HTTP POST requests, but how can I do it using SSL? I've googled around and haven't found anything that explains this well enough.
Here's the code I tried:
procedure TForm1.FormCreate(Sender: TObject);
var
responseXML:TMemoryStream;
responseFromServer:string;
begin
responseXML := TMemoryStream.Create;
IdSSLIOHandlerSocketOpenSSL1 := TIdSSLIOHandlerSocketOpenSSL.Create(self);
with idSSLIOHandlerSocketOpenSSL1 do
begin
SSLOptions.Method := sslvSSLv2;
SSLOptions.Mode := sslmUnassigned;
SSLOptions.VerifyMode := [];
SSLOptions.VerifyDepth := 0;
host := '';
end;
IdHTTP1 := TIdHTTP.Create(Self);
with IdHTTP1 do
begin
IOHandler := IdSSLIOHandlerSocketOpenSSL1;
AllowCookies := True;
ProxyParams.BasicAuthentication := False;
ProxyParams.ProxyPort := 0;
Request.ContentLength := -1;
Request.ContentRangeEnd := 0;
Request.ContentRangeStart := 0;
Request.Accept := 'text/html, */*';
Request.BasicAuthentication := False;
Request.UserAgent := 'Mozilla/3.0 (compatible; Indy Library)';
HTTPOptions := [hoForceEncodeParams];
end;
responsefromserver := IdHTTP1.Post('https://.../','name1=value1&name2=value2&....');
end;
When I try to run it I get the following error:
Project myProject.exe raised exception class EFOpenError with message 'Cannot open file "C:\...\Projects\Debug\Win32\name1=value1name2=value2 The system cannot find the file specified'.
I don't understand that. I sent parameters, though the errors sounds like I would have sent a file.
Also I have included libeay32.dll and ssleay32.dll within my myProject.exe folder.
You didn't specified your Delphi version or indy version, but I had some problems before with the bundled Indy with Delphi 2009 and HTTPS, and when I got the latest source from indy svn, the problem solved.
http://chee-yang.blogspot.com/2008/03/using-indy-https-client-to-consume.html
var S: TStringList;
M: TStream;
begin
S := TStringList.Create;
M := TMemoryStream.Create;
try
S.Values['Email'] := 'your google account';
S.Values['Passwd'] := 'your password';
S.Values['source'] := 'estream-sqloffice-1.1.1.1';
S.Values['service'] := 'cl';
IdHTTP1.IOHandler := IdSSLIOHandlerSocketOpenSSL1;
IdHTTP1.Request.ContentType := 'application/x-www-form-urlencoded';
IdHTTP1.Post('https://www.google.com/accounts/ClientLogin', S, M);
Memo1.Lines.Add(Format('Response Code: %d', [IdHTTP1.ResponseCode]));
Memo1.Lines.Add(Format('Response Text: %s', [IdHTTP1.ResponseText]));
M.Position := 0;
S.LoadFromStream(M);
Memo1.Lines.AddStrings(S);
finally
S.Free;
M.Free;
end;
end;
So the short answer is simple, use a COM-Object with flexible late binding, example of a translate service with language detection, implements in maXbox script:
function getPostTranslateLibre(feedstream: string): string;
var
Url,API_KEY, source: string;
jo, locate: TJSONObject;
httpReq,hr: Olevariant;
strm: TStringStream;
begin
httpReq:= CreateOleObject('WinHttp.WinHttpRequest.5.1');
// Open the HTTPs connection.
try
hr:= httpReq.Open('POST','https://libretranslate.pussthecat.org/detect', false);
httpReq.setRequestheader('user-agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0');
httpReq.setRequestheader('content-type','application/x-www-form-urlencoded');
//httpReq.setRequestheader('X-RapidAPI-Host','nlp-translation.p.rapidapi.com');
//httpReq.setRequestheader('X-RapidAPI-Key','...333');
if hr= S_OK then HttpReq.Send('q='+HTTPEncode(feedstream));
/// Send HTTP Post Request & get Responses.
If HttpReq.Status = 200 Then
result:= HttpReq.responseText
Else result:= 'Failed at getting response:'+itoa(HttpReq.Status)+HttpReq.responseText;
//writeln('debug response '+HttpReq.GetAllResponseHeaders);
finally
httpreq:= unassigned;
end;
end;
As early binding compiled we use LHttpClient: TALWininetHttpClient;
https://github.com/infussolucoes/usercontrol-sd/blob/master/Source/Terceiros/Alcinoe/ALHttpClient.pas
Another alternative to Indy is Synapse.
This class library offers full control of the post, but also offers a simple one liner post method as well:
function HttpPostURL(const URL, URLData: string; const Data: TStream): Boolean;