Garbage in headers when POST-ing with Indy 10.5.8 - delphi

Question

I'm trying to send a file using POST in multipart/form data via Indy 10.5.8. I'm using Delphi XE2 and I've been trying to POST a file to a server. This is the firs time I've tried this and since my experience with Indy is quite limited, i took the following snippet of code:
unit MsMultiPartFormData;
interface
uses
SysUtils, Classes;
const
CONTENT_TYPE = 'multipart/form-data; boundary=';
CRLF = #13#10;
CONTENT_DISPOSITION = 'Content-Disposition: form-data; name="%s"';
FILE_NAME_PLACE_HOLDER = '; filename="%s"';
CONTENT_TYPE_PLACE_HOLDER = 'Content-Type: %s' + crlf + crlf;
CONTENT_LENGTH = 'Content-Length: %d' + crlf;
type
TMsMultiPartFormDataStream = class(TMemoryStream)
private
FBoundary: string;
FRequestContentType: string;
function GenerateUniqueBoundary: string;
public
procedure AddFormField(const FieldName, FieldValue: string);
procedure AddFile(const FieldName, FileName, ContentType: string; FileData: TStream); overload;
procedure AddFile(const FieldName, FileName, ContentType: string); overload;
procedure PrepareStreamForDispatch;
constructor Create;
property Boundary: string read FBoundary;
property RequestContentType: string read FRequestContentType;
end;
implementation
{ TMsMultiPartFormDataStream }
constructor TMsMultiPartFormDataStream.Create;
begin
inherited;
FBoundary := GenerateUniqueBoundary;
FRequestContentType := CONTENT_TYPE + FBoundary;
end;
procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName,
ContentType: string; FileData: TStream);
var
sFormFieldInfo: string;
Buffer: PChar;
iSize: Int64;
begin
iSize := FileData.Size;
sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION +
FILE_NAME_PLACE_HOLDER + CRLF + CONTENT_LENGTH +
CONTENT_TYPE_PLACE_HOLDER, [FieldName, FileName, iSize, ContentType]);
{so: boundary + crlf + content-disposition+file-name-place-holder}
Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
FileData.Position := 0;
GetMem(Buffer, iSize);
try
FileData.Read(Buffer^, iSize);
Write(Buffer^, iSize);
finally
FreeMem(Buffer, iSize);
end;
end;
procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName,
ContentType: string);
var
FileStream: TFileStream;
begin
FileStream := TFileStream.Create(FileName, fmOpenRead or fmShareDenyWrite);
try
AddFile(FieldName, FileName, ContentType, FileStream);
finally
FileStream.Free;
end;
end;
procedure TMsMultiPartFormDataStream.AddFormField(const FieldName,
FieldValue: string);
var
sFormFieldInfo: string;
begin
sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION + CRLF + CRLF +
FieldValue, [FieldName]);
Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
end;
function TMsMultiPartFormDataStream.GenerateUniqueBoundary: string;
begin
Result := '---------------------------' + FormatDateTime('mmddyyhhnnsszzz', Now);
end;
procedure TMsMultiPartFormDataStream.PrepareStreamForDispatch;
var
sFormFieldInfo: string;
begin
sFormFieldInfo := CRLF + '--' + Boundary + '--' + CRLF;
Write(Pointer(sFormFieldInfo)^, Length(sFormFieldInfo));
Position := 0;
end;
end.
I'm calling the code like that:
function PostFile(filename, apikey: string): boolean;
var
ResponseStream: TMemoryStream;
MultiPartFormDataStream: TMsMultiPartFormDataStream;
begin
// Form5.IdHTTP1.HandleRedirects := true;
Form5.idHTTP1.ReadTimeout := 0;
// Form5.idHTTP1.IOHandler.LargeStream := True;
Result := false;
MultiPartFormDataStream := TMsMultiPartFormDataStream.Create;
ResponseStream := TMemoryStream.Create;
try
try
Form5.IdHttp1.Request.ContentType := MultiPartFormDataStream.RequestContentType;
MultiPartFormDataStream.AddFormField('apikey', apikey);
MultiPartFormDataStream.AddFile('file', filename, 'multipart/form-data');
MultiPartFormDataStream.PrepareStreamForDispatch;
MultiPartFormDataStream.Position := 0;
Form5.IdHTTP1.Post('http://www.updserver.tld/api//file/save', MultiPartFormDataStream, ResponseStream);
MultiPartFormDataStream.SaveToFile(ExtractFilePath(Application.ExeName) + 'a.txt');
Result := true;
except
on E:Exception do
begin
Form5.Close;
ShowMessage('Upload failed! '+E.Message);
end;
end;
finally
MultiPartFormDataStream.Free;
ResponseStream.Free;
end;
end;
The file gets sent, but is rejected by the server. Closer inspection of the data sent reveals that the data gets somewhat corrupt (i suspect encoding issues) - what I see is:
POST /api/file/save HTTP/1.0
Connection: keep-alive
Content-Type: multipart/form-data; boundary=---------------------------071312151405662
Content-Length: 11040172
Host: www.updserver.tld
Accept: text/html, */*
Accept-Encoding: identity
User-Agent: Mozilla/3.0 (compatible; Indy Library)
.
.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.0.7.1.3.1.2.1.5.1.4.0.5.6.6.2.
.
.C.o.n.t.e.n.t.-.D.i.s.p.o.s.i.t.i.o.n.:. .f.o.r.m.-.d.a.t.a.;. .n
.
.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.0.7.1.3.1.2.1.5.1.4.0.5.6.6.2.
.
.C.o.n.t.e.n.t.-.D.i.s.p.o.s.i.t.i.o.n.:. .f.o.r.m.-.d.a.t.a.;. .n.a.m.e.=.".f.i.l.e.".;. .f.i.l.e.n.a.m.e.=.".C.:.\.U.s.e........................>.......................................................v.......:...;...<.......[.......v.......................t.......o.......z............
...
...
The regular headers, sent from a working Python client, look like this:
POST https://updserver.tld/api/file/save HTTP/1.0
content-type: multipart/form-data; boundary=---------------------------071312151405662
content-length: 6613364
---------------------------071312151405662
Content-Disposition: form-data; name="apikey"
ac36fae9a406596[rest-of-api-key-goes-here]17966c42b60c8c4cd
---------------------------071312151405662
Content-Disposition: form-data; name="file"; filename="C:\Users\User\Desktop\Project1.exe"
Content-Type: application/octet-stream
Any idea about what I'm doing wrong?
Thanks in advance.

Answer 1

The root of the problem is that your custom TStream code is not compatible with D2009+ versions of Delphi. Delphi's String and PChar types are not Ansi anymore, but the code assumes they still are. They are Unicode UTF-16 now. You are not accounting for that correctly, eg:
procedure TMsMultiPartFormDataStream.AddFile(const FieldName, FileName, ContentType: string; FileData: TStream);
var
sFormFieldInfo: AnsiString;
iSize: Int64;
begin
iSize := FileData.Size;
// NOTE: this will only work for ASCII filenames!!!!
//
// Non-ASCII filenames will get converted to Ansi, which can cause data loss.
// To send non-ASCII filenames correctly, you have to encode it to a charset
// first, such as UTF-8, and then encode the resulting bytes using
// MIME's RFC2047 encoding so the server can decode the filename properly
// on its end...
//
sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION +
FILE_NAME_PLACE_HOLDER + CRLF + CONTENT_LENGTH +
CONTENT_TYPE_PLACE_HOLDER, [FieldName, FileName, iSize, ContentType]);
{so: boundary + crlf + content-disposition+file-name-place-holder}
Write(sFormFieldInfo[1], Length(sFormFieldInfo) * SizeOf(AnsiChar));
if iSize > 0 then
begin
FileData.Position := 0;
CopyFrom(FileData, iSize);
end;
end;
procedure TMsMultiPartFormDataStream.AddFormField(const FieldName, FieldValue: string);
var
sFormFieldInfo: AnsiString;
begin
// NOTE: this will only work for ASCII text!!!!
//
// Non-ASCII text will get converted to Ansi, which can cause data loss.
// To send non-ASCII text correctly, you have to encode it to a charset
// first, such as UTF-8 and then encode the resulting bytes using
// MIME's 'quoted-printable' or 'base64' enoding, and then include
// appropriate 'charset' and Content-Transfer-Encoding' headers so the
// server can decode the data properly on its end...
//
sFormFieldInfo := Format(CRLF + '--' + Boundary + CRLF + CONTENT_DISPOSITION + CRLF + CRLF +
FieldValue, [FieldName]);
Write(sFormFieldInfo[1], Length(sFormFieldInfo) * AnsiString(AnsiChar));
end;
procedure TMsMultiPartFormDataStream.PrepareStreamForDispatch;
var
sFormFieldInfo: AnsiString;
begin
sFormFieldInfo := CRLF + '--' + Boundary + '--' + CRLF;
Write(sFormFieldInfo[1], Length(sFormFieldInfo) * SizeOf(AnsiChar));
Position := 0;
end;
With that said, I strongly suggest you get rid of your custom TMsMultiPartFormDataStream class completely. All it is doing is mimicing an outdated version of Indy's own TIdMultipartFormDataStream class . Just use Indy's native TIdMultipartFormDataStream class as-is instead. It handles D2009+ Unicode for you, eg:
uses
..., IdMultipartFormData;
function PostFile(const filename, apikey: string): boolean;
var
ResponseStream: TMemoryStream;
MultiPartFormDataStream: TIdMultiPartFormDataStream;
begin
Result := False;
//Form5.IdHTTP1.HandleRedirects := true;
Form5.idHTTP1.ReadTimeout := 0;
//Form5.idHTTP1.IOHandler.LargeStream := True;
try
ResponseStream := TMemoryStream.Create;
try
MultiPartFormDataStream := TIdMultiPartFormDataStream.Create;
try
MultiPartFormDataStream.AddFormField('apikey', apikey);
MultiPartFormDataStream.AddFile('file', filename, 'application/octet-stream');
Form5.IdHTTP1.Post('http://www.updserver.tld/api/file/save', MultiPartFormDataStream, ResponseStream);
ResponseStream.SaveToFile(ExtractFilePath(Application.ExeName) + 'a.txt');
Result := True;
finally
MultiPartFormDataStream.Free;
end;
finally
ResponseStream.Free;
end;
except
on E:Exception do
begin
Form5.Close;
ShowMessage('Upload failed! ' + E.Message);
end;
end;
end;
end;

Answer 2

Do all of those . characters represent a 00 byte? Because that looks like the result of a ASCII->UTF16 conversion. And the garbage in the Content-Disposition could be related to a conversion like that copying bytes around and getting the end of the buffer wrong, leaving you with a corrupted string.
If you can come up with code to reproduce this consistently, you should probably post a bug report on the Indy forums.