I run task for importing dates to DB Mysql. After running task rake db:restore displayed error:
cannot open %=ENV[C9_USER]%: No such file
What wrong?
task:
require 'yaml'
namespace :db do
def backup_prep
#directory = File.join(Rails.root, 'db', 'backup')
#db = YAML::load( File.open( File.join(Rails.root, 'config', 'database.yml') ) )[ Rails.env ]
#db_params = "-u #{#db['username']} #{#db['database']}"
#db_params = "-p#{#db['password']} #{#db_params}" unless #db['password'].blank?
end
desc 'Backup database by mysqldump'
task :backup => :environment do
backup_prep
FileUtils.mkdir #directory unless File.exists?(#directory)
file = File.join( #directory, "#{RAILS_ENV}_#{DateTime.now.to_s}.sql" )
command = "mysqldump #{#db_params} | gzip > #{file}.gz" #--opt --skip-add-locks
puts "dumping to #{file}..."
# p command
exec command
end
desc "restore most recent mysqldump (from db/backup/*.sql.*) into the current environment's database."
task :restore => :environment do
unless ENV['RAILS_ENV']=='development'
puts "Are you sure you want to import into #{ENV['RAILS_ENV']}?! [y/N]"
return unless STDIN.gets =~ /^y/i
end
backup_prep
wildcard = File.join( #directory, ENV['FILE'] || "#{ENV['FROM']}*.sql*" )
puts file = `ls -t #{wildcard} | head -1`.chomp # default to file, or most recent ENV['FROM'] or just plain most recent
if file =~ /\.gz(ip)?$/
command = "gunzip < #{file} | mysql #{#db_params}"
else
command = "mysql #{#db_params} < #{file}"
end
p command
puts "please wait, this may take a minute or two..."
exec command
end
end
dump db store in path workspace/db/backup/db.sql
Related
I am doing Automation testing running JRuby with ConEmu to put files from my Filezila Client to my local host. It should not be to the remote server. Getting errors when I try running the below code in Ruby for establishing the connection. Transferring files here are not as important.
APOLOGIES: Will fix soon.
require 'net-ssh'
require 'net-sftp'
require 'dir'
local_path = 'D:\Rubynetssh'
remote_path = '/cguclaim/virtual/data/logs/gwlogs/ClaimCenter/'
file_perm = 0644
dir_perm = 0755
puts 'Connecting to remote server'
Net::SSH.start('server', 'admin', 'password1') do |ssh|
ssh.sftp.connect do |sftp|
puts 'Checking for files which need updating'
Find.find(local_path) do |file|
next if File.stat(file).directory?
local_file = "#{dir}/#{file}"
remote_file = remote_path + local_file.sub(local_path, '')
begin
remote_dir = File.dirname(remote_file)
sftp.stat(remote_dir)
rescue Net::SFTP::Operations::StatusException => e
raise unless e.code == 2
sftp.mkdir(remote_dir, :permissions => dir_perm)
end
begin
rstat = sftp.stat(remote_file)
rescue Net::SFTP::Operations::StatusException => e
raise unless e.code == 2
sftp.put_file(local_file, remote_file)
sftp.setstat(remote_file, :permissions => file_perm)
next
end
if File.stat(local_file).mtime > Time.at(rstat.mtime)
puts "Copying #{local_file} to #{remote_file}"
sftp.put_file(local_file, remote_file)
end
end
end
puts 'Disconnecting from remote server'
end
puts 'File transfer complete'
When I run the command below
jruby net-sftp.rb
this results in this error syntax
SyntaxError: net-sftp.rb:48: syntax error, unexpected tCONSTANT puts 'File transfer complete'
* EDIT *
***Now that you put in the code in the comment it comes up as error as shown below:
1. LoadError: no such file to load -- net-ssh
2. require at org/jruby/RubyKernel.java:939
3. require at
4. C:/jruby-9.0.4.0/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:54
5. <top> at net-sftp.rb:1
The following code written below worked for me better to establish the connection from Filezilla to ConEmu but now I need to get into the ClaimCenter folder to view the files if they exist or not
require "net/ssh"
require "net/sftp"
#hostname = "server"
#username = "admin"
#password = "password1"
#cmd = "ls -la"
res = ""
ssh = Net::SSH.start(#hostname, #username, password: #password) do |ssh|
res = ssh.exec!(#cmd)
end
puts res
The script is in the file fxrates_scraper.rb, code below.
The scheduler i'm using is in the schedule.rb file, code below.
I'm using the whenever gem, and when i type in 'bundle exec whenever' i see the task as scheduled, it seems fine. but it doesn't work: it isn't executing the script.
what am I doing wrong? note i haven't setup capistrano or anything, i just want the script to run on my local drive. also: when i execute the script manually, it runs just fine. it's just the automation bit that isn't working.
FXRATES_SCRAPER.RB
require 'open-uri'
require 'nokogiri'
require 'csv'
# Store URL to be scraped
url = "https://deliveroo.co.uk/restaurants/london/maida-vale?postcode=W92DE"
# Parse the page with Nokogiri
page = Nokogiri::HTML(open(url))
# Display output onto the screen
name =[]
page.css('span.list-item-title.restaurant-name').each do |line|
name << line.text.strip
end
category = []
page.css('span.restaurant-detail.detail-cat').each do |line|
category << line.text.strip
end
delivery_time = []
page.css('span.restaurant-detail.detail-time').each do |line|
delivery_time << line.text.strip
end
distance = []
page.css('span.restaurant-detail.detail-distance').each do |line|
distance << line.text.strip
end
status = []
page.css('li.restaurant--details').each do |line|
if line.attr("class").include? "unavailable"
sts = "closed"
else
sts = "open"
end
status << sts
end
# Write data to CSV file
CSV.open("deliveroo.csv", "w") do |file|
file << ["Name", "Category", "Delivery Time", "Distance", "Status"]
name.length.times do |i|
file << [name[i], category[i], delivery_time[i], distance[i], status[i]]
end
end
SCHEDULE.RB
set :output, "#{path}/log/cron.log"
every 2.minutes do
command "ruby '#{path}/fxrates_scraper.rb'"
end
OUTPUT FROM RUNNING 'BUNDLE EXEC WHENEVER' COMMAND
Faisals-Air:whatrate fkhalid2008$ bundle exec whenever
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58 * * * * /bin/bash -l -c 'ruby '\''/Users/fkhalid2008/whatrate/fxrates_scraper.rb'\'' >> /Users/fkhalid2008/whatrate/log/cron.log 2>&1'
## [message] Above is your schedule file converted to cron syntax; your crontab file was not updated.
Running 'whenever -i' command fixed this.
IMO, you have to set environment when run whenever:
bundle exec whenever --update-crontab --set environment='development'
Sometimes we do a COPY from our Production Database to our Quality Database. When this happens I just create a new database and link it in the Rails Application.
Question: Can you empty an entire postgresql Database including the Relationships and import the new DB?
Problem: When importing the database it does not overwrite the current data / structure / relations ...
Info:
I export the Production database like this:
## Dump without user privileges
pg_dump -x -U database1 -h localhost -O database1 > database1.sql
And normally I import the exported database like this:
## Import
psql -U database2 database2 < database1.sql
I've used the following rake task for a number of years and it's worked well for me. The local:db:abort_if_active_connections dependency isn't strictly necessary but it's nice as otherwise the backup fails since you can't drop a database currently in use.
You'll want to tweak the local:backup:production task's system commands to be whatever you need to get a copy of the database. Then you can just run:
bin/rake local:backup:production
lib/tasks/local/backup.rake
namespace :local do
namespace :backup do
desc 'Backup production and restore to development'
task :production => ['production:db']
namespace :production do
desc 'Backup production database and restore to development'
task :db => ['local:db:abort_if_active_connections', 'db:drop', 'db:create'] do
config = ActiveRecord::Base.configurations[Rails.env]
gz_file = "#{ActiveRecord::Base.configurations['production']['database']}.gz"
puts "Copying production database to temporary file on this machine..."
system "scp user#example.com:/tmp/#{gz_file} /tmp/#{gz_file}"
puts "Recreating local database..."
system "gunzip -c /tmp/#{gz_file} | psql #{config['database']}"
puts "Removing temporary file..."
File.unlink "/tmp/#{gz_file}"
end
end
end
end
lib/tasks/local.rake
namespace :local do
namespace :db do
task :abort_if_active_connections => ['db:load_config'] do
config = ActiveRecord::Base.configurations[Rails.env]
ActiveRecord::Base.establish_connection(config.merge('database' => 'postgres', 'schema_search_path' => 'public'))
version = ActiveRecord::Base.connection.send(:postgresql_version)
if version >= 90200
pid = 'pid'
else
pid = 'procpid'
end
connections = ActiveRecord::Base.connection.select_all("SELECT * FROM pg_stat_activity WHERE pg_stat_activity.datname = '#{config['database']}' AND #{pid} <> #{$$}")
unless connections.empty?
puts
puts "There are active connections to the database '#{config['database']}':"
puts
puts "%-7s %-20s %-16s %-20s %s" % %w[pid usename client_addr application_name backend_start]
connections.each do |c|
puts "%-7s %-20s %-16s %-20s %s" % [c[pid], c['usename'], c['client_addr'], c['application_name'], c['backend_start']]
end
puts
exit 1
end
ActiveRecord::Base.clear_all_connections!
end
end
end
Reffering that answer I was trying to use OptionParser to parse rake arguments. I simplified example from there and I had to add two ARGV.shift to make it work.
require 'optparse'
namespace :user do |args|
# Fix I hate to have here
puts "ARGV: #{ARGV}"
ARGV.shift
ARGV.shift
puts "ARGV: #{ARGV}"
desc 'Creates user account with given credentials: rake user:create'
# environment is required to have access to Rails models
task :create => :environment do
options = {}
OptionParser.new(args) do |opts|
opts.banner = "Usage: rake user:create [options]"
opts.on("-u", "--user {username}","Username") { |user| options[:user] = user }
end.parse!
puts "user: #{options[:user]}"
exit 0
end
end
This is the output:
$ rake user:create -- -u foo
ARGV: ["user:create", "--", "-u", "foo"]
ARGV: ["-u", "foo"]
user: foo
I assume ARGV.shift is not the way it should be done. I would like to know why it doesn't work without it and how to fix it in a proper way.
You can use the method OptionParser#order! which returns ARGV without the wrong arguments:
options = {}
o = OptionParser.new
o.banner = "Usage: rake user:create [options]"
o.on("-u NAME", "--user NAME") { |username|
options[:user] = username
}
args = o.order!(ARGV) {}
o.parse!(args)
puts "user: #{options[:user]}"
You can pass args like that: $ rake foo:bar -- '--user=john'
I know this does not strictly answer your question, but did you consider using task arguments?
That would free you having to fiddle with OptionParser and ARGV:
namespace :user do |args|
desc 'Creates user account with given credentials: rake user:create'
task :create, [:username] => :environment do |t, args|
# when called with rake user:create[foo],
# args is now {username: 'foo'} and you can access it with args[:username]
end
end
For more info, see this answer here on SO.
Try this:
require 'optparse'
namespace :programs do |args|
desc "Download whatever"
task :download => [:environment] do
# USAGE: rake programs:download -- rm
#-- Setting options $ rake programs:download -- --rm
options = {}
option_parser = OptionParser.new
option_parser.banner = "Usage: rake programs:download -- rm"
option_parser.on("-r", "--rm","Remove s3 files downloaded") do |value|
options[:remove] = value
end
args = option_parser.order!(ARGV) {}
option_parser.parse!(args)
#-- end
# your code here
end
end
You have to put a '=' between -u and foo:
$ rake user:create -- -u=foo
Instead of:
$ rake user:create -- -u foo
I setup a series of custom rake tasks for a project that I am building that are name spaced to placewise. These tasks can be viewed by running rake tasks from the console.
desc 'List all available placewise rake tasks for this application.'
task :tasks do
result = %x[rake -T | sed -n '/placewise:/{/grep/!p;}']
result.each_line do |task|
puts task
end
end
All of these tasks are stored in lib/tasks/placewise and built like so:
namespace :placewise do
namespace :db do
desc "Drop and create the current database, the argument [env] = environment."
task :recreate, [:env] do |t,args|
env = environments(args.env)
msg("Dropping the #{env} database")
shell("RAILS_ENV=#{env} rake db:drop", step: "1/3")
msg("Creating the #{env} database")
shell("RAILS_ENV=#{env} rake db:create", step: "2/3")
msg("Running the #{env} database migrations")
shell("RAILS_ENV=#{env} rake db:migrate", step: "3/3")
end
end
end
A new task, for example may start with the base setup as follows:
namespace :placewise do
namespace :example do
desc "example"
task :example do
end
end
end
As you can see the namespace :placewise do will be replicated each time. I want to keep all of our custom rake tasks in the same group, however, I am curious if there is a way to avoid having to add that namespace to each .rake file?
Cheers.
Unfortunately I was advised against this strategy and in the process of discovery I found out my helper methods were not setup correctly. So here goes.
I created a new modules folder in lib/modules with the new helper_functions.rb file inside that directory. Here are my helpers:
module HelperFunctions
# ------------------------------------------------------------------------------------
# Permitted environments
# ------------------------------------------------------------------------------------
def environments(arg)
arg = arg || "development"
environments = ["development", "test", "production"]
if environments.include?(arg)
puts
msg("Environment parameter is valid")
return arg
else
error("Invalid environment parameter")
exit
end
end
# ------------------------------------------------------------------------------------
# Console message handler
# ------------------------------------------------------------------------------------
def msg(txt, periods: "yes", new_line: "yes")
txt = txt + "..." if periods == "yes"
puts "===> " + txt
puts if new_line == "yes"
end
def error(reason)
puts "**** ERROR! ABORTING: " + reason + "!"
end
# ------------------------------------------------------------------------------------
# Execute Shell Commands
# ------------------------------------------------------------------------------------
def shell(cmd, step: nil)
msg("Starting step #{step}", new_line: "no") if step.present?
if ENV['TRY']
puts "-->> " + cmd
else
sh %{#{cmd}}
end
msg("#{step} completed!", periods: "no")
end
end
Then in the Rakefile add:
# Shared Ruby functions used in rake tasks
require File.expand_path('../lib/modules/helper_functions', __FILE__)
include HelperFunctions
Rails.application.load_tasks
# Do not add other tasks to this file, make files in the primary lib/tasks dir ending in .rake
# All placewise tasks should be under the lib/tasks/placewise folder and end in .rake
desc 'List all available placewise rake tasks for this application.'
task :tasks do
result = %x[rake -T | sed -n '/placewise:/{/grep/!p;}']
result.each_line do |task|
puts task
end
end
And finally my .rake tasks look like:
namespace :placewise do
# ------------------------------------------------------------------------------------
namespace :db do
# ------------------------------------------------------------------------------------
desc "Drop and create the current database, the argument [env] = environment."
task :recreate, [:env] do |t,args|
env = HelperFunctions::environments(args.env)
HelperFunctions::msg("Dropping the #{env} database")
HelperFunctions::shell("RAILS_ENV=#{env} rake db:drop", step: "1/3")
HelperFunctions::msg("Creating the #{env} database")
HelperFunctions::shell("RAILS_ENV=#{env} rake db:create", step: "2/3")
HelperFunctions::msg("Running the #{env} database migrations")
HelperFunctions::shell("RAILS_ENV=#{env} rake db:migrate", step: "3/3")
end
# ------------------------------------------------------------------------------------
end
# ------------------------------------------------------------------------------------
end