Zero downtime restart of unicorn/ rails - ruby-on-rails

In our deploy scripts we use the following snippet to restart unicorn:
desc "Zero downtime restart of Unicorn"
task :restart do
run "kill -s USR2 unicorn_pid"
end
The master process forks, starts the new workers and then kills the old one. But now it seems the new master kills the old master and takes over any new connections before the new children are fully up. As we disabled preloading of the app using preload_app false the new workers take around 30 - 60 seconds to start. During this time the new connections/ the website hangs. How to avoid this, so making the new master only take over once the new children are fully up and ready to server requests? :)
Update:
My unicorn.rb looks like this:
# name of application
application = "myapp"
# environment specific stuff
case ENV["RAILS_ENV"]
when "integration", "staging"
worker_processes 1
when "production"
worker_processes 4
else
raise "Invalid runtime environment '#{ENV["RAILS_ENV"]}'"
end
# set directories
base_dir = "/srv/#{application}/current"
shared_path = "/srv/#{application}/shared"
working_directory base_dir
# maximum runtime of requests
timeout 300
# multiple listen directives are allowed
listen "#{shared_path}/unicorn.sock", :backlog => 64
# location of pid file
pid "#{shared_path}/pids/unicorn.pid"
# location of log files
stdout_path "#{shared_path}/log/unicorn.log"
stderr_path "#{shared_path}/log/unicorn.err"
# combine REE with "preload_app true" for memory savings
# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
preload_app false
if GC.respond_to?(:copy_on_write_friendly=)
GC.copy_on_write_friendly = true
end
before_exec do |server|
# see http://unicorn.bogomips.org/Sandbox.html
ENV["BUNDLE_GEMFILE"] = "#{base_dir}/Gemfile"
end
before_fork do |server, worker|
# the following is highly recomended for "preload_app true"
if defined?(ActiveRecord::Base)
ActiveRecord::Base.connection.disconnect!
end
if defined?(Sequel::Model)
Sequel::DATABASES.each{ |db| db.disconnect }
end
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
# someone else did our job for us
end
end
end
after_fork do |server, worker|
if defined?(ActiveRecord::Base)
ActiveRecord::Base.establish_connection
end
end
I think the main problem is that there's no child_ready hook. The before_fork and after_hook are called "too early". I think I could add some logic in the after_fork hook to somehow detect when a child is ready...but I hope there's an easier solution? :)

Related

RuntimeError: The connection cannot be reused in the forked process (ruby-oci8 gem)

I am getting
RuntimeError: The connection cannot be reused in the forked process
from
ruby-oci8 (2.1.3) lib/oci8/cursor.rb:28:in `__initialize'
I am using oracle db for rails application, recently I started deploying rails application with unicorn and nginx and from their onward I am getting this error please help.
I am using ruby 1.9.3 with rails 3.1 and this is my unicorn.rb file
rails_env = ENV['RAILS_ENV'] || 'production'
worker_processes 6
preload_app true
timeout 75
app_dir = File.expand_path("../../..", __FILE__)
shared_path = "#{app_dir}/shared"
working_directory "#{app_dir}/current"
# Set up socket location
listen "#{shared_path}/sockets/unicorn.sock", :backlog => 64
# Set master PID location
pid "#{shared_path}/pids/unicorn.pid"
stderr_path "#{shared_path}/log/unicorn.log"
stdout_path "#{shared_path}/log/unicorn.log"
before_fork do |server, worker|
Signal.trap 'TERM' do
puts 'Unicorn master intercepting TERM and sending myself QUIT instead'
Process.kill 'QUIT', Process.pid
end
defined?(ActiveRecord::Base) and
ActiveRecord::Base.connection.disconnect!
end
after_fork do |server, worker|
# reset the connection since the pre-forked connection will be stale
Signal.trap 'TERM' do
puts 'Unicorn worker intercepting TERM and doing nothing. Wait for master to send QUIT'
end
if defined?(ActiveRecord::Base)
config = ActiveRecord::Base.configurations[Rails.env] ||
Rails.application.config.database_configuration[Rails.env]
config['pool'] = ENV['DB_POOL'] || 5
ActiveRecord::Base.establish_connection(config)
end
end
I got the solution. Basically my model contains the establish_connection definition like this
class User < ActiveRecord::Base
establish_connection "user_#{Rails.env}"
end
and my unicorn.rb file before_fork block contains the following code.
defined?(ActiveRecord::Base) and
ActiveRecord::Base.connection.disconnect!
I found that above code doesn't work when you explicitly write establish_connection in your model. To resolve the issue I had to change the above code to the below one
defined?(ActiveRecord::Base) &&
ActiveRecord::Base.remove_connection(User) &&
ActiveRecord::Base.connection.disconnect!
and it worked like a charm. :-)

Rails 4. Activerecord validation rules and callbacks are being disabled after unicorn restart

I use Ruby v2.2.3, Rails v4.2.4, Unicorn v5.1.0 in production.
Activerecord validation rules and callbacks for all models of app has been disabled after unicorn restart (from log rotation script). Here it is
/var/www/fitness_crm/shared/log/production.log
/var/www/fitness_crm/shared/log/unicorn.stderr.log
/var/www/fitness_crm/shared/log/unicorn.stdout.log {
daily
missingok
rotate 30
compress
notifempty
create 640 deploy deploy
sharedscripts
postrotate
kill -s USR2 `cat /var/www/fitness_crm/shared/tmp/pids/unicorn.pid`
endscript
}
We've changed the script to send USR1 instead USR2, but nothing has changed. When USR1 is sent validations/callbacks simply keep being disabled. Here is our unicorn configuration file
working_directory "/var/www/fitness_crm/current"
pid "/var/www/fitness_crm/shared/tmp/pids/unicorn.pid"
stdout_path "/var/www/fitness_crm/shared/log/unicorn.stdout.log"
stderr_path "/var/www/fitness_crm/shared/log/unicorn.stderr.log"
listen "/tmp/unicorn.fitness_crm_production.sock"
worker_processes 8
timeout 30
preload_app true
before_exec do |server|
ENV["BUNDLE_GEMFILE"] = "/var/www/fitness_crm/current/Gemfile"
end
before_fork do |server, worker|
# Disconnect since the database connection will not carry over
if defined? ActiveRecord::Base
ActiveRecord::Base.connection.disconnect!
end
# Quit the old unicorn process
old_pid = "#{server.config[:pid]}.oldbin"
if File.exists?(old_pid) && server.pid != old_pid
begin
Process.kill("QUIT", File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
# someone else did our job for us
end
end
if defined?(Resque)
Resque.redis.quit
end
sleep 1
end
after_fork do |server, worker|
# Start up the database connection again in the worker
if defined?(ActiveRecord::Base)
ActiveRecord::Base.establish_connection
end
if defined?(Resque)
Resque.redis = 'localhost:6379'
end
end
After this we kill unicorn and start it manually with command:
bundle exec unicorn -D -c /var/www/fitness_crm/shared/config/unicorn.rb -E production
After this everything is good and validations and callbacks are enabled again.
Please, help to find out what is the reason of such a behavior and how to fix it.

Currently my unicorn master needs to be killed manually before each deployment

I'm using capistrano in my rails4 app and everytime I'm release a new version (=> cap production deploy) I need to kill the unicorn master on the server so that the capistrano can walk without any failures.
### How can I automate the process of killing the unicorn process?
This is how my deploy.rb is looking like:
lock '3.4.0'
set :application, 'maalify'
set :repo_url, 'git#github.com:iNeedCode/Maalify.git'
set :deploy_to, '/opt/www/maalify'
set :user, 'root'
set :linked_dirs, %w{log tmp/pids tmp/cache tmp/sockets}
set :rbenv_ruby, '2.2.1'
set :rbenv_type, :user
set :rbenv_path, "~/.rbenv"
set :rbenv_prefix, "RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec"
set :rbenv_map_bins, %w(rake gem bundle ruby rails)
set :rbenv_roles, :all
set :linked_files, %w{config/database.yml .rbenv-vars} # create these files manually ones on the server
# Capristrano3 unicorn
set :unicorn_pid, "/opt/www/maalify/current/shared/tmp/pids/unicorn.pid"
set :unicorn_config_path, "/opt/www/maalify/current/config/unicorn.rb"
# Clean up all older releases
before :deploy, "unicorn:stop"
after "deploy:publishing", "unicorn:start"
after "deploy:restart", "deploy:cleanup"
namespace :deploy do
after :restart, :clear_cache do
on roles(:web), in: :groups, limit: 3, wait: 10 do
execute :rake, 'cache:clear'
end
end
end
Update
Here is the Unicorn.rb
# set path to application
app_dir = "/opt/www/maalify"
shared_dir = "#{app_dir}/shared"
working_directory "#{app_dir}/current"
# Set unicorn options
worker_processes 1
preload_app true
timeout 30
# Set up socket location
listen "#{shared_dir}/tmp/sockets/unicorn.sock", :backlog => 64
# Logging
stderr_path "#{shared_dir}/log/unicorn.stderr.log"
stdout_path "#{shared_dir}/log/unicorn.stdout.log"
# Set master PID location
pid "#{shared_dir}/tmp/pids/unicorn.pid"
After adding the before_fork I'm still getting the same error as previously when I didn't kill unicorn by hand:
/opt/www/maalify/shared/bundle/ruby/2.2.0/gems/unicorn-4.8.3/lib/unicorn/http_server.rb:206:in `pid=': Already running on PID:16268 (or pid=/opt/www/maalify/shared/tmp/pids/unicorn.pid is stale) (ArgumentError)
from /opt/www/maalify/shared/bundle/ruby/2.2.0/gems/unicorn-4.8.3/lib/unicorn/http_server.rb:135:in `start'
from /opt/www/maalify/shared/bundle/ruby/2.2.0/gems/unicorn-4.8.3/bin/unicorn:126:in `<top (required)>'
from /opt/www/maalify/shared/bundle/ruby/2.2.0/bin/unicorn:23:in `load'
from /opt/www/maalify/shared/bundle/ruby/2.2.0/bin/unicorn:23:in `<main>'
I've noticed in your unicorn config file next directive:
preload_app true
Here is info from docs:
HUP - reloads config file and gracefully restart all workers. If the
"preload_app" directive is false (the default), then workers will also
pick up any application code changes when restarted. If "preload_app"
is true, then application code changes will have no effect; USR2 +
QUIT (see below) must be used to load newer code in this case.
You can set directive preload_app to false and then if you need to reload code, send HUP signal to master process.
But here is another way to reload code without downtime. You need to implement USR2 + QUIT method in config. It means when you send USR2 signal, unicorn starts new instance of application without killing old one. You can hook before_fork callback to which will kill old instance painless for you.
Here is example, add this to unicorn config:
before_fork do |server, worker|
ActiveRecord::Base.connection.disconnect!
old_pid = "#{server.config[:pid]}.oldbin"
if File.exists?(old_pid) && server.pid != old_pid
begin
Process.kill("QUIT", File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
# someone else did our job for us
end
end
end
after_fork do |server, worker|
ActiveRecord::Base.establish_connection
end
Now send USR2 signal to master process and check it out!
Next step is change your deploy.rb. You need to remove all unicorn hooks and add next one:
after 'deploy:publishing', 'deploy:restart'
namespace :deploy do
task :restart do
invoke 'unicorn:legacy_restart'
end
end
And last step: fix pid path in deploy.rb.
set :unicorn_pid, "/opt/www/maalify**/current/shared/**tmp/pids/unicorn.pid"
Must be:
set :unicorn_pid, "/opt/www/maalify/shared/tmp/pids/unicorn.pid"

Running Faye in a different environment than Rails

I'm trying to setup Rails 4.2.1 and Faye with CSRF protection. I used the guide at http://faye.jcoglan.com/security/csrf.html to get everything working.
config/csrf_protection.rb
class CsrfProtection
def incoming(message, request, callback)
session_token = request.session['_csrf_token']
message_token = message['ext'] && message['ext'].delete('csrfToken')
unless session_token == message_token
message['error'] = '401::Access denied'
end
callback.call(message)
end
end
config/application.rb
config.middleware.insert_after ActionDispatch::Session::CookieStore,
Faye::RackAdapter,
:extensions => [CsrfProtection.new],
:mount => '/live',
:timeout => 25
config/unicorn.rb
worker_processes Integer(ENV["WEB_CONCURRENCY"] || 3)
timeout 15
preload_app true
listen 3000, tcp_nopush: false
stderr_path "log/unicorn.stderr.log"
stdout_path "log/unicorn.stdout.log"
before_fork do |server, worker|
Signal.trap 'TERM' do
puts 'Unicorn master intercepting TERM and sending myself QUIT instead'
Process.kill 'QUIT', Process.pid
end
defined?(ActiveRecord::Base) and
ActiveRecord::Base.connection.disconnect!
end
after_fork do |server, worker|
Signal.trap 'TERM' do
puts 'Unicorn worker intercepting TERM and doing nothing. Wait for master to send QUIT'
end
defined?(ActiveRecord::Base) and
ActiveRecord::Base.establish_connection
end
And I start unicorn with:
bundle exec unicorn -c config/unicorn.rb
I'm getting the following errors in my log file:
Rack::Lint::LintError: Status must be >=100 seen as integer
ThreadError: deadlock; recursive locking
The errors are being caused by Faye running in development mode. Is it possible, using this setup, to tell Faye to run in a production environment while letting the rest of my application run in development mode?

Unicorn does not sever database connections properly after new deploy

I am currently having a problem with database connections after a Capistrano deploy. On the database server it seems like unicorn does not sever the previous connections and it just keeps adding on top of the old ones. I am doing a preload true, and I also have octopus gem installed as well if that matters. I am not too sure who is to blame for this. I have pasted my unicorn config for the part that matters. Any help is appreciated!
before_fork do |server, worker|
if defined?(ActiveRecord::Base)
ActiveRecord::Base.connection.disconnect!
end
old_pid = "/tmp/unicorn.my_app_name.pid.oldbin"
if File.exists?(old_pid) && server.pid != old_pid
begin
Process.kill("QUIT", File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
end
after_fork do |server, worker|
ActiveRecord::Base.connection_proxy.instance_variable_get(:#shards).each {|k,v| v.clear_reloadable_connections! }
if defined?(ActiveRecord::Base)
ActiveRecord::Base.establish_connection
end
if Process.ppid > 1 # Not the daemon
child_pid = server.config[:pid].sub(".pid", ".#{worker.nr}.pid")
File.open(child_pid, "wb") {|f| f << Process.pid }
end
end
"ps aux"
Your "New Theory" is correct. 2 workers with 5 connections will result in 10 total.
Answer is here - Unicorn do not close DB connections

Resources