The request was aborted error while writing to request stream - asp.net-mvc

Question

public HttpWebResponse PushFileToWistia(byte[] contentFileByteArray, string fileName)
{
StringBuilder postDataBuilder = new StringBuilder();
postDataBuilder.Append("I am appending all the wistia config and setting here");
byte[] postData = null;
using (MemoryStream postDataStream = new MemoryStream())
{
byte[] postDataBuffer = Encoding.UTF8.GetBytes(postDataBuilder.ToString());
postDataStream.Write(postDataBuffer, 0, postDataBuffer.Length);
postDataStream.Write(contentFileByteArray, 0, contentFileByteArray.Length);
postDataBuffer = Encoding.UTF8.GetBytes("\r\n--" + boundary + "--");
postDataStream.Write(postDataBuffer, 0, postDataBuffer.Length);
postData = postDataStream.ToArray();
}
ServicePointManager.Expect100Continue = false;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(AppConfig.WistiaCustomCourseBucket);
request.Method = "POST";
request.Expect = String.Empty;
request.Headers.Clear();
request.ContentType = "multipart/form-data; boundary=" + boundary;
request.ContentLength = postData.Length;
Stream requestStream = request.GetRequestStream();
requestStream.Write(postData, 0, postData.Length); //for file > 100mb this call throws and error --the requet was aborted. the request was canceled.
requestStream.Flush();
requestStream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
return response;
}
The above code works for video file mp4 less then 50mb. But when I try to upload a 100mb file it throws and exception (Request was aborted.) I need to support file size up to 1.5gb So now I am not sure if this approach is correct for such a big file size upload. Any suggestions in the right direction will be helpful...thanks(I am trying to upload the file to Wistia Server)
The exception is thrown at this line
-- requestStream.Write(postData, 0, postData.Length);
I have tried changing the web.config setting but didn't work:
httpRuntime targetFramework="4.5" maxRequestLength="2048576" executionTimeout="12000" requestLengthDiskThreshold="1024"
------Async Call-------
MemoryStream wistiaFileStream = null;
using (MemoryStream postDataStream = new MemoryStream())
{
postDataStream.Write(contentFileByteArray, 0, contentFileByteArray.Length);
wistiaFileStream = postDataStream;
postDataStream.Flush();
postDataStream.Close();
}
Stream requestStream = await request.GetRequestStreamAsync();
await requestStream.WriteAsync(wistiaMetadata, 0, wistiaMetadata.Length);
using (wistiaFileStream)
{
byte[] wistiaFileBuffer = new byte[500*1024];
int wistiaFileBytesRead = 0;
while (
(wistiaFileBytesRead =
await wistiaFileStream.ReadAsync(wistiaFileBuffer, 0, wistiaFileBuffer.Length)) != 0)
{
await requestStream.WriteAsync(wistiaFileBuffer, 0, wistiaFileBytesRead);
await requestStream.FlushAsync();
}
await requestStream.WriteAsync(requestBoundary, 0, requestBoundary.Length);
}

Answer 1

I would suggest moving to async and write file directly from file system to request in order to avoid triple buffering of 1.5GB in memory (warning below is not tested).
public async Task<HttpWebResponse> PushFileToWistiaAsync(string contentFilePath)
{
string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
string contentBoundary = "\r\n--" + boundary + "\r\n";
StringBuilder wistiaMetadataBuilder = new StringBuilder();
wistiaMetadataBuilder.Append("--" + boundary + "\r\n");
// Append all the wistia config and setting here
byte[] wistiaMetadata = Encoding.UTF8.GetBytes(wistiaMetadataBuilder.ToString());
byte[] requestBoundary = Encoding.UTF8.GetBytes(contentBoundary);
ServicePointManager.Expect100Continue = false;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(AppConfig.WistiaCustomCourseBucket);
request.Method = "POST";
request.Headers.Clear();
request.Expect = String.Empty;
request.ContentType = "multipart/form-data; boundary=" + boundary;
Stream requestStream = await request.GetRequestStreamAsync();
await requestStream.WriteAsync(wistiaMetadata, 0, wistiaMetadata.Length);
using (FileStream wistiaFileStream = new FileStream(contentFilePath, FileMode.Open, FileAccess.Read))
{
byte[] wistiaFileBuffer = new byte[500 * 1024];
int wistiaFileBytesRead = 0;
while ((wistiaFileBytesRead = await wistiaFileStream.ReadAsync(wistiaFileBuffer, 0, wistiaFileBuffer.Length)) != 0)
{
await requestStream.WriteAsync(wistiaFileBuffer, 0, wistiaFileBytesRead);
await requestStream.FlushAsync();
}
}
await requestStream.WriteAsync(requestBoundary, 0, requestBoundary.Length);
return (HttpWebResponse)(await request.GetResponseAsync());
}
You should play with buffer sizes, amount of data you read at once and request.SendChunked to achieve reasonable performance.
Here is another approach (not asynchronous so possibly worst scalability) which wirtes directly from buffer to request:
public HttpWebResponse PushFileToWistia(byte[] contentFileByteArray)
{
string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
string contentBoundary = "\r\n--" + boundary + "\r\n";
StringBuilder wistiaMetadataBuilder = new StringBuilder();
wistiaMetadataBuilder.Append("--" + boundary + "\r\n");
// Append all the wistia config and setting here
byte[] wistiaMetadata = Encoding.UTF8.GetBytes(wistiaMetadataBuilder.ToString());
byte[] requestBoundary = Encoding.UTF8.GetBytes(contentBoundary);
ServicePointManager.Expect100Continue = false;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(AppConfig.WistiaCustomCourseBucket);
request.Method = "POST";
request.Headers.Clear();
request.Expect = String.Empty;
request.ContentType = "multipart/form-data; boundary=" + boundary;
request.ContentLength = wistiaMetadata.Length + contentFileByteArray.Length + requestBoundary.Length
// You can play with SendChunked and AllowWriteStreamBuffering to control the size of chunks you send and performance
//request.SendChunked = true;
//request.AllowWriteStreamBuffering = false;
int contentFileChunkSize = 500 * 1024;
int contentFileBytesRead = 0;
Stream requestStream = request.GetRequestStream();
requestStream.Write(wistiaMetadata, 0, wistiaMetadata.Length);
while (contentFileBytesRead < contentFileByteArray.Length)
{
if ((contentFileBytesRead + contentFileChunkSize) > contentFileByteArray.Length)
{
contentFileChunkSize = contentFileByteArray.Length - contentFileBytesRead;
}
requestStream.Write(contentFileByteArray, contentFileBytesRead, contentFileChunkSize);
requestStream.Flush();
contentFileBytesRead += contentFileChunkSize;
}
requestStream.Write(requestBoundary, 0, requestBoundary.Length);
requestStream.Close();
// You might need to play with request.Timeout here
return (HttpWebResponse)request.GetResponse();
}
Also if you doing this in web application and you want to use asynchronous approach you need to "async/await" all the way up (so async action in async controller etc.).
In general I would discourage doing this as part of request handling in web application (the total time observed from user perspective would be a sum of uploading to your app and then to Wistia which might be much more than client timeout allows). In such case it is usually better to save the file and schedule some other "background task" to do the upload job.