Rails upload large file - ruby-on-rails

Question

I ran into some difficulties while uploading larges files using rails 5.
File is uploaded using ajax and simply grab inside a rails controller.
Used server is : puma.
The file transfer is speed (followed by ajax xhr progress, in a local network (Gigabit)).
But the save of the file /tmp/RackMultipart* took a long time.
I suppose that the file is loaded in memory by Rack, that process and save it in /tmp/. After that, the controller is proceed.
The code work perfectly for small files like images.
But for large file > 100 Mo the completed execution take around 1 minute ...
My code :
The upload area :
views/_attachments.html.erb
<div class="card">
<div class="card-header">
Fichiers
</div>
<div class="card-block">
<span id="attachment-area-message"></span>
<div id="attachment-area">
Déposez vos fichiers ici
</div>
<!-- Area for progress bar -->
<div id="progress-wrapper"></div>
<script>
var attachment_token = '<%= form_authenticity_token %>';
var attachment_model_name = '<%= fileable.class.name %>';
var attachment_model_id = '<%= fileable.id %>';
</script>
</div>
<div class="card-block">
<div class="attachfiles-wrapper">
<div id="attachfiles">
<% fileable.attachments.includes('user').order(created_at: :asc).each do |attachment| %>
<%= render partial: 'app/attachments/attachment', locals: { attachment: attachment } %>
<% end %>
</div>
</div>
</div>
</div>
JS file which launch the upload :
$(document).on('turbolinks:load', function() {
new Clipboard('.btn-clipboard');
var upload_mime = [
'application/zip',
// Image
'image/png',
'image/jpeg',
'image/gif',
'image/tiff',
'image/svg+xml',
];
var upload_maxSize = 3000000000;
var server_url = '/app/attachments/upload.js'; // Route for upload file, .js for the js call back
var element = $("#attachment-area");
// EVENTS
// ----------------------------------------------------------------------------
element.on('dragover', function(e) {
e.preventDefault();
e.stopPropagation();
});
element.on('dragenter', function(e) {
element.addClass('active');
e.preventDefault();
e.stopPropagation();
});
element.on('dragleave', function(e) {
element.removeClass('active');
e.preventDefault();
e.stopPropagation();
});
element.on('drop', function(e) {
element.removeClass('active');
e.preventDefault();
e.stopPropagation();
if (e.originalEvent.dataTransfer){
if (e.originalEvent.dataTransfer.files.length > 0) {
console.log(e.originalEvent.dataTransfer.files);
upload(e.originalEvent.dataTransfer.files);
}
}
return false;
});
// UPLOADS
// ----------------------------------------------------------------------------
var upload = function(files) {
// Send each file
$.each(files, function(key, file) {
// TEST THE FILE
// ----------------------
var FileValidate = true;
// Size
if(file.size > upload_maxSize) {
$('#attachment-area-message').append(file.name + " : Fichier trop lourd (3 Go maximum) : " + file.size);
FileValidate = false;
}
// Mime type
if( upload_mime.indexOf(file.type) == -1 ) {
$('#attachment-area-message').append( file.name + " : Type de fichier non authorisé : " + file.type);
$('#attachment-area-message').append( "<br>Essayez de zipper le fichier");
FileValidate = false;
}
if(!FileValidate) return true; // Continue to next iteration
// SEND FILE
// ----------------------
console.log(file);
var formData = new FormData();
formData.append('attachment[file]', file );
formData.append("attachment[model_name]", attachment_model_name);
formData.append("attachment[model_id]", attachment_model_id);
console.log(formData);
// Progress Bar Name
var progress_name = file.name.replace(/[^a-zA-Z]/g,'-').toLowerCase();
// Send the request :)
$.ajax({
url: server_url,
data: formData,
type: 'POST',
beforeSend: function(request) {
request.setRequestHeader('X-CSRF-Token', attachment_token);
console.log('BEFORE SEND');
},
contentType: false, // NEEDED, DON'T OMIT THIS (requires jQuery 1.6+)
processData: false, // NEEDED, DON'T OMIT THIS
xhr: function() {
// create an XMLHttpRequest
var xhr = new XMLHttpRequest();
console.log('xhr');
xhr.upload.onprogress = function (e) {
console.log('xhr progress');
if (e.lengthComputable) {
var percente = Math.round( ( e.loaded * 100 ) / e.total );
$('.' + progress_name + ' .progress-bar').width(percente + "%");
}
};
xhr.onloadstart = function (e) {
console.log('xhr onloadstart');
$('#progress-wrapper').append('<div class="' + progress_name + '" style="margin-top:5px;">'
+ '<span class="description">' + file.name + '</span>'
+ '<div class="progress" id="file-upload-bar">'
+ '<div class="progress-bar bg-info" role="progressbar" style="width:0%; height:10px;" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100"></div>'
+ '</div></div>');
};
xhr.onload = function (e) {
console.log('xhr onload');
if (xhr.status === 200) eval(xhr.responseText); // Grab the return of rails controller (format.js)
};
xhr.onloadend = function (e) {
console.log('xhr onloadend');
$('.' + progress_name).remove();
};
return xhr;
}
});
});
};
});
And the controller :
(Attachment model is polymorphic base on fileable).
class App::AttachmentsController < AppController
before_action :find_fileable
def upload
# Get the File
uploaded_io = attach_params[:file]
logger.debug '---------'
logger.debug params.inspect
logger.debug '---------'
# Define file destination
dest = Rails.root.join('public', 'uploads', 'attachments', attach_params[:model_name], attach_params[:model_id], uploaded_io.original_filename)
file_name = uploaded_io.original_filename
file_basename = File.basename(uploaded_io.original_filename, '.*')
file_extname = File.extname(uploaded_io.original_filename)
# Make dir
dir = File.dirname( dest )
FileUtils.mkdir_p(dir) unless File.directory?(dir)
# Test if file exist (and update version if needed)
if File.exist?(dest)
version = 0
loop do
version += 1
file_name = file_basename + '-' + version.to_s + file_extname
dest = Rails.root.join('public', 'uploads', 'attachments', attach_params[:model_name], attach_params[:model_id], file_name )
break if !File.exist?(dest)
end
end
# Copy file to dest
#FileUtils.cp uploaded_io.path, dest
File.open( dest, 'wb') do |file|
file.write(uploaded_io.read)
end
# Save in database
#attach = #fileable.attachments.new
#attach.user_id = #current_user.id
#attach.name = file_name
#attach.size = uploaded_io.size
#attach.mime = uploaded_io.content_type
#attach.key = Digest::SHA1.hexdigest([Time.now, rand].join)
respond_to do |format|
if #attach.save
flash[:success] = "Fichier ajouté"
format.js # upload.js callback add new file to the list of files
else
flash[:warning] = "Fichier non enregistré :("
end
end
end
private
def attach_params
params.require( :attachment ).permit( :model_id, :model_name, :file )
end
def find_fileable
#fileable = Task.find_by_id( attach_params[:model_id] ) if attach_params[:model_name] == 'Task'
end
end
I have tested different file management solutions : CarrierWave, Shrine, ...
Unfortunately the problem is still there. Always the rack save in front.
Any help or any idea ? I want to eat this "Rack"
Thanks,
Seb.

Answer 1

The issue is that Rack multipart parser writes to disk (that's the fast part), but that its implementation is slow and expensive (see rack/rack#1075). However, the Rack master has huge performance improvements to the multipart parser, so using master should solve your problem.
gem "rack", github: "rack/rack"
We can verify this by running the following script:
require "rack"
require "rack/test_app" # https://github.com/kwatch/rack-test_app
require "benchmark"
app = -> (env) do
puts Benchmark.realtime { Rack::Request.new(env).params } # trigger multipart parsing
[200, {}, []]
end
File.write("file.txt", "a" * 100*1024*1024)
test_app = Rack::TestApp.wrap(app)
test_app.post("/", multipart: {file: File.open("file.txt")})
Rack 2.0.3:
$ ruby multipart.rb
62.617582999984734
Rack master:
$ ruby multipart.rb
0.3564810000243597

Answer 2

I have test with chunk method. Create part of 1 Mo of my file and send them in binary. It's better but not the perfection.
With this method rails do not create a MultiRack* file in tmp, but it use memory in both sides, server and client.
The javascript file :
$(document).on('turbolinks:load', function() {
new Clipboard('.btn-clipboard');
var upload_url = '/app/attachments/upload'; // Route for upload file, .js for the js call back
var upload_part_url = '/app/attachments/upload/part/';
var upload_mime = [
'application/zip',
// Vidéo
'video/mp4',
'video/mpeg',
'video/x-flv',
// Audio
'audio/mpeg',
// Image
'image/png',
'image/jpeg',
'image/gif',
'image/tiff',
'image/svg+xml',
// Text
'text/csv',
'text/html',
// Application
'application/pdf',
'application/msword',
'application/excel',
'application/mspowerpoint',
// Adobe
'application/vnd.adobe.indesign',
'application/x-indesign',
'application/indesign',
'image/vnd.adobe.photoshop',
'application/x-photoshop',
'application/photoshop',
'application/psd',
'image/psd',
'application/illustrator',
'application/postscript'
];
var upload_maxSize = 3000000000;
// EVENTS on DROP AREA
// ----------------------------------------------------------------------------
var element = $("#attachment-area"); // Drop area
element.on('dragover', function(e) {
e.preventDefault();
e.stopPropagation();
});
element.on('dragenter', function(e) {
element.addClass('active');
e.preventDefault();
e.stopPropagation();
});
element.on('dragleave', function(e) {
element.removeClass('active');
e.preventDefault();
e.stopPropagation();
});
element.on('drop', function(e) {
element.removeClass('active');
e.preventDefault();
e.stopPropagation();
if (e.originalEvent.dataTransfer){
if (e.originalEvent.dataTransfer.files.length > 0) {
// We upload the files
$.each(e.originalEvent.dataTransfer.files, function(key, file) {
// Test the file
var FileValidate = true;
// Size
if(file.size > upload_maxSize) {
$('#attachment-area-message').append(file.name + " : Fichier trop lourd (3 Go maximum) : " + file.size);
FileValidate = false;
}
// Mime type
if( upload_mime.indexOf(file.type) == -1 ) {
$('#attachment-area-message').append( file.name + " : Type de fichier non authorisé : " + file.type);
$('#attachment-area-message').append( "<br>Essayez de zipper le fichier");
FileValidate = false;
}
// Begin the upload
if(FileValidate) upload(file);
});
}
}
return false;
});
// UPLOAD
// ----------------------------------------------------------------------------
var upload = function(file) {
console.log(file);
var formData = new FormData();
formData.append("attachment[model_name]", attachment_model_name);
formData.append("attachment[model_id]", attachment_model_id);
formData.append('attachment[file_name]', file.name );
formData.append('attachment[file_size]', file.size );
formData.append('attachment[file_mime]', file.type );
// Progress Bar Name
// var progress_name = file.name.replace(/[^a-zA-Z]/g,'-').toLowerCase();
// Send the file infos
var req = new XMLHttpRequest();
// Request events
req.upload.onprogress = function (e) {
console.log('xhr progress');
};
req.onloadstart = function (e) {
console.log('xhr onloadstart');
};
// Error
req.onerror = function (e) {
}
// Success
req.onload = function (e) {
console.log('xhr onload');
if (req.status === 200) {
attach = JSON.parse(req.responseText);
if(typeof attach.id !== 'undefined') uploadFileData(file, attach.id ); // Send the data
}
};
// Complete
req.onloadend = function (e) {
console.log('xhr onloadend');
};
// Send the file infos Request
req.open("POST", upload_url);
req.setRequestHeader('X-CSRF-Token', attachment_token);
req.send(formData);
};
// UPLOAD FILE CHUNKS
// ----------------------------------------------------------------------------
var uploadFileData = function(file, id) {
var reader = new FileReader();
// Process after the file is read
reader.onload = function (e) {
var chunkSize = 1*1024*1024;
var buffer = this.result;
var fileSize = buffer.byteLength;
var segments = Math.ceil(fileSize / chunkSize);
var count = 0;
var fileId = id;
// Send part
(function sendPart() {
var segSize = Math.min(chunkSize, fileSize - count * chunkSize);
var returnFormat = segSize < chunkSize ? '.js' : '.json' ;
if (segSize > 0) {
var chunk = new Uint8Array(buffer, count++ * chunkSize, segSize); // get a chunk
// update progress bar
var req = new XMLHttpRequest();
// Request events
req.upload.onprogress = function (e) {
console.log('part progress : ' + count );
};
req.onloadstart = function (e) {
console.log('part onloadstart : ' + count );
};
// Error
req.onerror = function (e) {
}
// Success
req.onload = function (e) {
console.log('part next : ' + count );
sendPart(); // Success -> Next part
};
// Send the file part data
req.open("POST", upload_part_url + fileId + returnFormat);
req.setRequestHeader('X-CSRF-Token', attachment_token);
req.setRequestHeader('Content-Type', 'application/octet-stream');
req.send(chunk);
}
else {
// hide progress bar
console.log("part Done : " + count);
}
})()
};
// Read the file
reader.readAsArrayBuffer(file);
reader.onprogress = function(e) {
// loaded += e.loaded;
// progress.value = (loaded/total) * 100;
};
}
});
The controller :
class App::AttachmentsController < AppController
before_action :find_fileable, only: [:upload]
def upload
# Define file destination
model_name = attach_params[:model_name]
model_id = attach_params[:model_id]
file_name = attach_params[:file_name]
file_basename = File.basename(file_name, '.*')
file_extname = File.extname(file_name)
dest = Rails.root.join('public', 'uploads', 'attachments', model_name, model_id, file_name)
# Make dir
dir = File.dirname( dest )
FileUtils.mkdir_p(dir) unless File.directory?(dir)
# Test if file exist (and update version in name if needed)
if File.exist?(dest)
version = 0
loop do
version += 1
file_name = file_basename + '-' + version.to_s + file_extname
dest = Rails.root.join('public', 'uploads', 'attachments', model_name, model_id, file_name )
break if !File.exist?(dest)
end
end
# Save in database
#a = #fileable.attachments.new
#a.user_id = #current_user.id
#a.name = file_name
#a.size = attach_params[:file_size]
#a.mime = attach_params[:file_mime]
#a.key = Digest::SHA1.hexdigest( [Time.now, rand].join )
#a.completed = false
#a.save
logger.debug '----'
logger.debug #a.to_json
logger.debug '----'
render status: 200, json: #a.to_json
end
def upload_part
#attach = Attachment.find(params[:id])
logger.debug '----'
logger.debug #attach.inspect
logger.debug '----'
dest = #attach.path
# Copy file to dest
File.open( dest, 'ab') do |file|
file.write(request.raw_post)
end
logger.debug '----'
logger.debug File.size(dest)
logger.debug #attach.size
logger.debug '----'
respond_to do |format|
format.js
format.json { render status: 200, json: { "status": "yop"} }
end
end
private
def attach_params
params.require( :attachment ).permit( :model_name, :model_id, :file_name, :file_size, :file_mime )
end
def find_fileable
#fileable = Task.find_by_id( attach_params[:model_id] ) if attach_params[:model_name] == 'Task'
end
end
See you in next episode. I keep searching ...