Related
In theory connection attempt should terminate after 5 seconds. However in practice I still get infinite timeout. Is there any way to terminate connection from outside. For example I could use timer and then execute proper function to abort. Another nasty workaround would be fire thread and kill it with TerminateThread function.
function DownloadFile(URL, {User, Pass,} FileName: string): Boolean;
const
BufferSize = 1024;
var
hSession, hURL: HInternet;
Buffer: array[1..BufferSize] of Byte;
BufferLen,Connect_timeout: DWORD;
F: File;
User,Pass:string;
begin
User:='';
Pass:='';
Result := False;
hSession := InternetOpen('', INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0) ;
Connect_timeout := 5000 ;
InternetSetOption(hSession, INTERNET_OPTION_CONNECT_TIMEOUT, Pointer(#Connect_timeout), sizeof(Connect_timeout));
// Establish the secure connection
InternetConnect (
hSession,
PChar(URL),
INTERNET_DEFAULT_HTTPS_PORT,
PChar(User),
PChar(Pass),
INTERNET_SERVICE_HTTP,
0,
0
);
try
hURL := InternetOpenURL(hSession, PChar(URL), nil, 0, 0, 0) ;
try
AssignFile(f, FileName);
Rewrite(f,1);
try
repeat
InternetReadFile(hURL, #Buffer, SizeOf(Buffer), BufferLen) ;
BlockWrite(f, Buffer, BufferLen)
until BufferLen = 0;
finally
CloseFile(f) ;
Result := True;
end;
finally
InternetCloseHandle(hURL)
end
finally
InternetCloseHandle(hSession)
end;
end;
I have one big text file with 95000+ lines on web
Now for example:
I have one string
I want to match that string in that file
and read that whole line where that string is placed in file
I am using wininet
I can load whole file in to string
but while matching it takes alot of time and gui hangs
is there anyway to do it more faster ?
edit:
Here is code i m using to get urlcontents from web
in button1
var
s:string;
begin
//1- as file is big it takes alot of time :/
S := GetUrlContent('web url example');
if (Pos('string i want to search', s)> 0) then
begin
mmo1.lines.add('Found');
//2- now how can i show extact line that contains string
end;
end;
//----------------------------------------------------
function GetUrlContent(const Url: string): string;
var
NetHandle: HINTERNET;
UrlHandle: HINTERNET;
Buffer: array[0..1024] of Char;
BytesRead: dWord;
begin
Result := '';
NetHandle := InternetOpen('Delphi 5.x', INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if Assigned(NetHandle) then
begin
UrlHandle := InternetOpenUrl(NetHandle, PChar(Url), nil, 0, INTERNET_FLAG_RELOAD, 0);
if Assigned(UrlHandle) then
{ UrlHandle valid? Proceed with download }
begin
FillChar(Buffer, SizeOf(Buffer), 0);
repeat
Result := Result + Buffer;
FillChar(Buffer, SizeOf(Buffer), 0);
InternetReadFile(UrlHandle, #Buffer, SizeOf(Buffer), BytesRead);
until BytesRead = 0;
InternetCloseHandle(UrlHandle);
end
else
{ UrlHandle is not valid. Raise an exception. }
raise Exception.CreateFmt('Cannot open URL %s', [Url]);
InternetCloseHandle(NetHandle);
end
else
{ NetHandle is not valid. Raise an exception }
raise Exception.Create('Unable to initialize');
end;
I got the source below from a third-party site explaining how to download a file from the internet using WinInet. I'm not too familiar with API, and I took at look at the WinInet unit but did not see any API calls like what I need.
What I'm doing is adding the ability to report back progress of downloading a file. This procedure I've already wrapped inside of a TThread and everything works fine. However, just one missing piece: Finding the total size of the source file before downloading.
See below where I have a comment //HOW TO GET TOTAL SIZE? This is where I need to find out what is the total size of the file BEFORE I begin downloading it. How do I go about doing this? Because this code seems to not know the size of the file until it's done being downloaded - and that makes this addition irrelevant.
procedure TInetThread.Execute;
const
BufferSize = 1024;
var
hSession, hURL: HInternet;
Buffer: array[1..BufferSize] of Byte;
BufferLen: DWORD;
f: File;
S: Bool;
D: Integer;
T: Integer;
procedure DoWork(const Amt: Integer);
begin
if assigned(FOnWork) then
FOnWork(Self, FSource, FDest, Amt, T);
end;
begin
S:= False;
try
try
if not DirectoryExists(ExtractFilePath(FDest)) then begin
ForceDirectories(ExtractFilePath(FDest));
end;
hSession:= InternetOpen(PChar(FAppName), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
try
hURL:= InternetOpenURL(hSession, PChar(FSource), nil, 0, 0, 0);
try
AssignFile(f, FDest);
Rewrite(f, 1);
T:= 0; //HOW TO GET TOTAL SIZE?
D:= 0;
DoWork(D);
repeat
InternetReadFile(hURL, #Buffer, SizeOf(Buffer), BufferLen);
BlockWrite(f, Buffer, BufferLen);
D:= D + BufferLen;
DoWork(D);
until BufferLen = 0;
CloseFile(f);
S:= True;
finally
InternetCloseHandle(hURL);
end
finally
InternetCloseHandle(hSession);
end;
except
on e: exception do begin
S:= False;
end;
end;
finally
if assigned(FOnComplete) then
FOnComplete(Self, FSource, FDest, S);
end;
end;
You can use the HEAD method and check the Content-Length to retrieve the file size of a remote file
Check these two Methods
WinInet
If you want execute a HEAD method you must use the HttpOpenRequest, HttpSendRequest and HttpQueryInfo WinInet functions .
uses
SysUtils,
Windows,
WinInet;
function GetWinInetError(ErrorCode:Cardinal): string;
const
winetdll = 'wininet.dll';
var
Len: Integer;
Buffer: PChar;
begin
Len := FormatMessage(
FORMAT_MESSAGE_FROM_HMODULE or FORMAT_MESSAGE_FROM_SYSTEM or
FORMAT_MESSAGE_ALLOCATE_BUFFER or FORMAT_MESSAGE_IGNORE_INSERTS or FORMAT_MESSAGE_ARGUMENT_ARRAY,
Pointer(GetModuleHandle(winetdll)), ErrorCode, 0, #Buffer, SizeOf(Buffer), nil);
try
while (Len > 0) and {$IFDEF UNICODE}(CharInSet(Buffer[Len - 1], [#0..#32, '.'])) {$ELSE}(Buffer[Len - 1] in [#0..#32, '.']) {$ENDIF} do Dec(Len);
SetString(Result, Buffer, Len);
finally
LocalFree(HLOCAL(Buffer));
end;
end;
procedure ParseURL(const lpszUrl: string; var Host, Resource: string);
var
lpszScheme : array[0..INTERNET_MAX_SCHEME_LENGTH - 1] of Char;
lpszHostName : array[0..INTERNET_MAX_HOST_NAME_LENGTH - 1] of Char;
lpszUserName : array[0..INTERNET_MAX_USER_NAME_LENGTH - 1] of Char;
lpszPassword : array[0..INTERNET_MAX_PASSWORD_LENGTH - 1] of Char;
lpszUrlPath : array[0..INTERNET_MAX_PATH_LENGTH - 1] of Char;
lpszExtraInfo : array[0..1024 - 1] of Char;
lpUrlComponents : TURLComponents;
begin
ZeroMemory(#lpszScheme, SizeOf(lpszScheme));
ZeroMemory(#lpszHostName, SizeOf(lpszHostName));
ZeroMemory(#lpszUserName, SizeOf(lpszUserName));
ZeroMemory(#lpszPassword, SizeOf(lpszPassword));
ZeroMemory(#lpszUrlPath, SizeOf(lpszUrlPath));
ZeroMemory(#lpszExtraInfo, SizeOf(lpszExtraInfo));
ZeroMemory(#lpUrlComponents, SizeOf(TURLComponents));
lpUrlComponents.dwStructSize := SizeOf(TURLComponents);
lpUrlComponents.lpszScheme := lpszScheme;
lpUrlComponents.dwSchemeLength := SizeOf(lpszScheme);
lpUrlComponents.lpszHostName := lpszHostName;
lpUrlComponents.dwHostNameLength := SizeOf(lpszHostName);
lpUrlComponents.lpszUserName := lpszUserName;
lpUrlComponents.dwUserNameLength := SizeOf(lpszUserName);
lpUrlComponents.lpszPassword := lpszPassword;
lpUrlComponents.dwPasswordLength := SizeOf(lpszPassword);
lpUrlComponents.lpszUrlPath := lpszUrlPath;
lpUrlComponents.dwUrlPathLength := SizeOf(lpszUrlPath);
lpUrlComponents.lpszExtraInfo := lpszExtraInfo;
lpUrlComponents.dwExtraInfoLength := SizeOf(lpszExtraInfo);
InternetCrackUrl(PChar(lpszUrl), Length(lpszUrl), ICU_DECODE or ICU_ESCAPE, lpUrlComponents);
Host := lpszHostName;
Resource := lpszUrlPath;
end;
function GetRemoteFileSize(const Url : string): Integer;
const
sUserAgent = 'Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101';
var
hInet : HINTERNET;
hConnect : HINTERNET;
hRequest : HINTERNET;
lpdwBufferLength: DWORD;
lpdwReserved : DWORD;
ServerName: string;
Resource: string;
ErrorCode : Cardinal;
begin
ParseURL(Url,ServerName,Resource);
Result:=0;
hInet := InternetOpen(PChar(sUserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if hInet=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetOpen Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hConnect := InternetConnect(hInet, PChar(ServerName), INTERNET_DEFAULT_HTTP_PORT, nil, nil, INTERNET_SERVICE_HTTP, 0, 0);
if hConnect=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetConnect Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hRequest := HttpOpenRequest(hConnect, PChar('HEAD'), PChar(Resource), nil, nil, nil, 0, 0);
if hRequest<>nil then
begin
try
lpdwBufferLength:=SizeOf(Result);
lpdwReserved :=0;
if not HttpSendRequest(hRequest, nil, 0, nil, 0) then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
if not HttpQueryInfo(hRequest, HTTP_QUERY_CONTENT_LENGTH or HTTP_QUERY_FLAG_NUMBER, #Result, lpdwBufferLength, lpdwReserved) then
begin
Result:=0;
ErrorCode:=GetLastError;
raise Exception.Create(Format('HttpQueryInfo Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hRequest);
end;
end
else
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hConnect);
end;
finally
InternetCloseHandle(hInet);
end;
end;
Indy
Also check this code using indy.
function GetRemoteFilesize(const Url :string) : Integer;
var
Http: TIdHTTP;
begin
Http := TIdHTTP.Create(nil);
try
Http.Head(Url);
result:= Http.Response.ContentLength;
finally
Http.Free;
end;
end;
Answering the question of how to get a download size with WinInet. This is out of one of my file downloaders that is based on WinInet.
This is the method I use to get the download size:
function TWebDownloader.GetContentLength(URLHandle: HINTERNET): Int64;
// returns the expected download size. Returns -1 if one not provided
var
SBuffer: Array[1..20] of char;
SBufferSize: Integer;
srv: integer;
begin
srv := 0;
SBufferSize := 20;
if HttpQueryInfo(URLHandle, HTTP_QUERY_CONTENT_LENGTH, #SBuffer, SBufferSize, srv) then
Result := StrToFloat(String(SBuffer))
else
Result := -1;
end;
Use of this method requires an open request handle, and does NOT require reading any of the data:
URLHandle := HttpOpenRequest(ConnectHandle, 'GET', Pchar(sitepath), nil,
nil, nil, INTERNET_FLAG_NO_CACHE_WRITE, 0);
...
DownloadSize := GetContentLength(URLHandle);
HTH
after fixing the types it looks better like this:
function GetContentLength(URLHandle:HINTERNET):Int64;
// returns the expected download size. Returns -1 if one not provided
var
SBufferSize, srv:Cardinal;
begin
srv:=0;
SBufferSize:=20;
if Not HttpQueryInfo(URLHandle, HTTP_QUERY_CONTENT_LENGTH or HTTP_QUERY_FLAG_NUMBER, {#SBuffer} #Result, SBufferSize, srv) then Result:=-1;
end;
to call it:
{get the file handle}
hURL:=InternetOpenURL(hSession, PChar(URL), nil, 0, 0, 0);
if hURL=Nil then
begin
InternetCloseHandle(hSession);
ShowMessage('The link is incorrect!');
exit;
end;
{get the file size}
filesize:=GetContentLength(hURL);
i need a file downloader component for Delphi . may you help me ?
Use the high-level URLDownloadToFile function:
uses UrlMon;
...
URLDownloadToFile(nil, 'http://www.rejbrand.se/', 'C:\Users\Andreas Rejbrand\Desktop\index.html', 0, nil);
Or, you could very easily write your own downloader function using the WinInet functions, something like
uses WinInet;
...
hInet := InternetOpen(PChar(UserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
try
hURL := InternetOpenUrl(hInet, PChar('http://' + Server + Resource), nil, 0, 0, 0);
try
repeat
InternetReadFile(hURL, #Buffer, SizeOf(Buffer), BufferLen);
...
There is a lot of sample code here at SO. Use the search box above.
Update
I wrote a small sample. You might want to execute this code in its own thread and let it ping back every 10 kB or so, so that you can provide the user with some progress bar, for instance.
function DownloadFile(const UserAgent, URL, FileName: string): boolean;
const
BUF_SIZE = 4096;
var
hInet, hURL: HINTERNET;
f: file;
buf: PByte;
amtc: cardinal;
amti: integer;
begin
result := false;
hInet := InternetOpen(PChar(UserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
try
hURL := InternetOpenUrl(hInet, PChar(URL), nil, 0, 0, 0);
try
GetMem(buf, BUF_SIZE);
try
FileMode := fmOpenWrite;
AssignFile(f, FileName);
try
Rewrite(f, 1);
repeat
InternetReadFile(hURL, buf, BUF_SIZE, amtc);
BlockWrite(f, buf^, amtc, amti);
until amtc = 0;
result := true;
finally
CloseFile(f);
end;
finally
FreeMem(buf);
end;
finally
InternetCloseHandle(hURL);
end;
finally
InternetCloseHandle(hInet);
end;
end;
You can also make this with Indy :
procedure DownloadHTTP(const AUrl : string; DestStream: TStream);
begin
with TIdHTTP.Create(Application) do
try
try
Get(AUrl,DestStream);
except
On e : Exception do
MessageDlg(Format('Erreur : %s',[e.Message]), mtInformation, [mbOK], 0);
end;
finally
Free;
end;
end;
If you want quick download, you can also use Clever Internet Suite
This question already has answers here:
How to send a HTTP POST Request in Delphi using WinInet api
(3 answers)
Closed 9 years ago.
I want to send a HTTP Post Request in Delphi 2010 using WinInet, but my script doesn't work ;/
It's my Delphi script:
uses WinInet;
procedure TForm1.Button1Click(Sender: TObject);
var
hNet,hURL,hRequest: HINTERNET;
begin
hNet := InternetOpen(PChar('User Agent'),INTERNET_OPEN_TYPE_PRECONFIG or INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if Assigned(hNet) then
begin
try
hURL := InternetConnect(hNet,PChar('http://localhost/delphitest.php'),INTERNET_DEFAULT_HTTP_PORT,nil,nil,INTERNET_SERVICE_HTTP,0,DWORD(0));
if(hURL<>nil) then
hRequest := HttpOpenRequest(hURL, 'POST', PChar('test=test'),'HTTP/1.0',PChar(''), nil, INTERNET_FLAG_RELOAD or INTERNET_FLAG_PRAGMA_NOCACHE,0);
if(hRequest<>nil) then
HttpSendRequest(hRequest, nil, 0, nil, 0);
InternetCloseHandle(hNet);
except
ShowMessage('error');
end
end;
end;
and my PHP script:
$data = $_POST['test'];
$file = "test.txt";
$fp = fopen($file, "a");
flock($fp, 2);
fwrite($fp, $data);
flock($fp, 3);
fclose($fp);
Major problems:
The second parameter of InternetConnect should contain only the name of the server, not the entire URL of the server-side script.
The third parameter of HttpOpenRequest should be the file name (URL) of the script, not the POST data!
The actual POST data should be the forth parameter of HttpSendRequest.
Minor problems
INTERNET_OPEN_TYPE_PRECONFIG or INTERNET_OPEN_TYPE_PRECONFIG: It is sufficient with INTERNET_OPEN_TYPE_PRECONFIG.
DWORD(0) is overkill. 0 is enough.
Sample Code
I use the following code to POST data:
procedure WebPostData(const UserAgent: string; const Server: string; const Resource: string; const Data: AnsiString); overload;
var
hInet: HINTERNET;
hHTTP: HINTERNET;
hReq: HINTERNET;
const
accept: packed array[0..1] of LPWSTR = (PChar('*/*'), nil);
header: string = 'Content-Type: application/x-www-form-urlencoded';
begin
hInet := InternetOpen(PChar(UserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
try
hHTTP := InternetConnect(hInet, PChar(Server), INTERNET_DEFAULT_HTTP_PORT, nil, nil, INTERNET_SERVICE_HTTP, 0, 1);
try
hReq := HttpOpenRequest(hHTTP, PChar('POST'), PChar(Resource), nil, nil, #accept, 0, 1);
try
if not HttpSendRequest(hReq, PChar(header), length(header), PChar(Data), length(Data)) then
raise Exception.Create('HttpOpenRequest failed. ' + SysErrorMessage(GetLastError));
finally
InternetCloseHandle(hReq);
end;
finally
InternetCloseHandle(hHTTP);
end;
finally
InternetCloseHandle(hInet);
end;
end;
For instance:
WebPostData('My UserAgent', 'www.rejbrand.se', 'mydir/myscript.asp', 'value=5');
Update in response to answer by OP
To read data from the Internet, use InternetReadFile function. I use the following code to read a small (one-line) text file from the Internet:
function WebGetData(const UserAgent: string; const Server: string; const Resource: string): string;
var
hInet: HINTERNET;
hURL: HINTERNET;
Buffer: array[0..1023] of AnsiChar;
i, BufferLen: cardinal;
begin
result := '';
hInet := InternetOpen(PChar(UserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
try
hURL := InternetOpenUrl(hInet, PChar('http://' + Server + Resource), nil, 0, 0, 0);
try
repeat
InternetReadFile(hURL, #Buffer, SizeOf(Buffer), BufferLen);
if BufferLen = SizeOf(Buffer) then
result := result + AnsiString(Buffer)
else if BufferLen > 0 then
for i := 0 to BufferLen - 1 do
result := result + Buffer[i];
until BufferLen = 0;
finally
InternetCloseHandle(hURL);
end;
finally
InternetCloseHandle(hInet);
end;
end;
Sample usage:
WebGetData('My UserAgent', 'www.rejbrand.se', '/MyDir/update/ver.txt')
This function thus only reads data, with no prior POST. However, the InternetReadFile function can also be used with a handle created by HttpOpenRequest, so it will work in your case also. You do know that the WinInet reference is MSDN, right? All Windows API functions are described in detail there, for instance InternetReadFile.
var
BufferIn : INTERNET_BUFFERS;
Buffer: array[0..1024] of Byte;
FTmp: TSomeStream:
FURL: string;
...
begin
... // Create FTmp, set FUrl, ...
NetHandle := InternetOpen( 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3',
INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
UrlHandle := HttpOpenRequest(NetHandle, 'POST', PChar(FURL), nil, nil, nil, INTERNET_FLAG_NO_CACHE_WRITE, 0);
... // Check handles, etc
try
BufferIn.dwBufferTotal := FTmp.Size;
if not HttpSendRequestEx(UrlHandle, #BufferIn, nil, 0, 0) then
raise Exception.CreateFmt('Error on HttpSendRequestEx %d\n', [GetLastError()]);
size := FTmp.Read(Buffer, 1024);
InternetWriteFile(UrlHandle, #Buffer, size, BytesWritten);
while (BytesWritten = size) and (BytesWritten > 0) do
begin
size := FTmp.Read(Buffer, 1024);
InternetWriteFile(UrlHandle, #Buffer, size, BytesWritten);
end;
finally
FTmp.Free;
InternetCloseHandle(UrlHandle);
InternetCloseHandle(NetHandle);
end;
There is a library (called 'HTTPGet component for Delphi 32') on http://www.utilmind.com. It installs a visual control into your component palette.
It does exactly what you want, so you may want to take a look.