I have a class which is currently being executed via delayed job. One of the tasks is to execute "rake spec" and redirect the output.
I do this as such:
class Executor
def execute_command(cmd, &block)
STDOUT.sync = true # That's all it takes...
IO.popen(cmd + " 2>&1") do |pipe| # Redirection is performed using operators
pipe.sync = true
while str = pipe.gets
block.call str # This is synchronous!
end
end
return $?.success?
end
end
However, none of the output appears and it doesn't even aware to execute the unit tests correctly.
Capistrano works and it works on OSX. My server is Ubuntu running Passenger.
Anyone have any ideas why the output wouldn't be redirecting?
Thanks
Ben
Try without the STDFDES redirection on cmd
here is what i used and what i get.
class Executor
def execute_command(cmd, &block)
STDOUT.sync = true # That's all it takes...
IO.popen(cmd) do |pipe| # Redirection is performed using operators
pipe.sync = true
while str = pipe.gets
block.call str # This is synchronous!
end
end
return $?.success?
end
end
Tested with:
ex = Executor.new
ex.execute_command "ps aux" do |str|
p str
end
result:
"USER PID %CPU %MEM VSZ RSS TT STAT STARTED
TIME COMMAND\n" "mitch 423 3.4 1.0 2750692 159552 ??
S 23Apr12 19:41.59 /Users/mitch/iTerm.app/Contents/MacOS/iTerm
-psn_0_40970\n" "_windowserver 90 3.1 1.8 3395124 301576 ?? Ss 20Apr12 75:19.86
/System/Library/Frameworks/ApplicationServices.framework/Frameworks/CoreGraphics.framework/Resources/WindowServer
-daemon\n" "mitch 78896 2.0 0.8 1067248 136088 ?? R Thu03PM 37:46.59 /Applications/Spotify.app/Contents/MacOS/Spotify
-psn_0_4900012\n" "mitch 436 1.8 1.0 1063952 169320 ?? S 23Apr12 100:23.87
/Applications/Skype.app/Contents/MacOS/Skype -psn_0_90134\n"
Related
when use forkmanager by ruby .it happen this:
ruby version:
ruby 2.4.1p111 (2017-03-22 revision 58053) [x64-mingw32]
system version:
windows7 64
Uncaught exception: fork() function is unimplemented on this machine
D:/Ruby24-x64/lib/ruby/gems/2.4.0/gems/parallel-forkmanager-2.0.1/lib/parallel/forkmanager.rb:525:in fork'
D:/Ruby24-x64/lib/ruby/gems/2.4.0/gems/parallel-forkmanager-
2.0.1/lib/parallel/forkmanager.rb:525:instart'
#!/usr/bin/env ruby
#encoding: UTF-8
# 抓取每一个站点的首页链接数量
require 'rubygems'
require 'ap'
require 'json'
require 'net/http'
require 'nokogiri'
require 'forkmanager'
require 'beanstalk-client'
class MultipleCrawler
class Crawler
def initialize(user_agent, redirect_limit=1)
#user_agent = user_agent
#redirect_limit = redirect_limit
#timeout = 20
end
attr_accessor :user_agent, :redirect_limit, :timeout
def fetch(website)
print "Pid:#{Process.pid}, fetch: #{website}\n"
redirect, url = #redirect_limit, website
start_time = Time.now
redirecting = false
begin
begin
uri = URI.parse(url)
req = Net::HTTP::Get.new(uri.path)
req.add_field('User-Agent', #user_agent)
res = Net::HTTP.start(uri.host, uri.port) do |http|
http.read_timeout = #timeout
http.request(req)
end
if res.header['location'] # 遇到重定向，则url设定为location，再次抓取
url = res.header['location']
redirecting = true
end
redirect -= 1
end while redirecting and redirect>=0
opened_time = (Time.now - start_time).round(4) # 统计打开网站耗时
encoding = res.body.scan(/<meta.+?charset=["'\s]*([\w-]+)/i)[0]
encoding = encoding ? encoding[0].upcase : 'GB18030'
html = 'UTF-8'==encoding ? res.body : res.body.force_encoding('GB2312'==encoding || 'GBK'==encoding ? 'GB18030' : encoding).encode('UTF-8')
doc = Nokogiri::HTML(html)
processed_time = (Time.now - start_time - opened_time).round(4) # 统计分析链接耗时, 1.8.7, ('%.4f' % float).to_f 替换 round(4)
[opened_time, processed_time, doc.css('a[#href]').size, res.header['server']]
rescue =>e
e.message
end
end
end
def initialize(websites, beanstalk_jobs, pm_max=1, user_agent='', redirect_limit=1)
#websites = websites # 网址数组
#beanstalk_jobs = beanstalk_jobs # beanstalk服务器地址和管道参数
#pm_max = pm_max # 最大并行运行进程数
#user_agent = user_agent # user_agent 伪装成浏览器访问
#redirect_limit = redirect_limit # 允许最大重定向次数
#ipc_reader, #ipc_writer = IO.pipe # 缓存结果的 ipc 管道
end
attr_accessor :user_agent, :redirect_limit
def init_beanstalk_jobs # 准备beanstalk任务
beanstalk = Beanstalk::Pool.new(*#beanstalk_jobs)
#清空beanstalk的残留消息队列
begin
while job = beanstalk.reserve(0.1)
job.delete
end
rescue Beanstalk::TimedOut
print "Beanstalk queues cleared!\n"
end
#websites.size.times{|i| beanstalk.put(i)} # 将所有的任务压栈
beanstalk.close
rescue => e
puts e
exit
end
def process_jobs # 处理任务
start_time = Time.now
pm = Parallel::ForkManager.new(#pm_max)
#pm_max.times do |i|
# 启动后，立刻 next 不会等待进程执行完，这样才可以并行运算
pm.start(i) and next
beanstalk = Beanstalk::Pool.new(*#beanstalk_jobs)
# 关闭读取管道，子进程只返回数据
#ipc_reader.close
loop{
begin
# 检测超时为0.1秒，因为任务以前提前压栈
job = beanstalk.reserve(0.1)
index = job.body
job.delete
website = #websites[index.to_i]
result = Crawler.new(#user_agent).fetch(website)
#ipc_writer.puts( ({website=>result}).to_json )
rescue Beanstalk::DeadlineSoonError, Beanstalk::TimedOut, SystemExit, Interrupt
break
end
}
#ipc_writer.close
pm.finish(0)
end
#ipc_writer.close
begin
# 等待所有子进程处理完毕
pm.wait_all_children
# 遇到中断，打印消息
rescue SystemExit, Interrupt
print "Interrupt wait all children!\n"
ensure
results = read_results
# 打印处理结果
ap results, :indent => -4 , :index=>false
print "Process end, total: #{#websites.size}, crawled: #{results.size}, time: #{'%.4f' % (Time.now - start_time)}s.\n"
end
end
def read_results # 通过管道读取子进程抓取返回的数据
results = {}
while result = #ipc_reader.gets
results.merge! JSON.parse(result)
end
#ipc_reader.close
results
end
def run # 运行入口
init_beanstalk_jobs
process_jobs
end
end
websites = %w(
http://www.51buy.com/ http://www.360buy.com/ http://www.tmall.com/ http://www.taobao.com/
http://china.alibaba.com/ http://www.paipai.com/ http://shop.qq.com/ http://www.lightinthebox.com/
http://www.amazon.cn/ http://www.newegg.com.cn/ http://www.vancl.com/ http://www.yihaodian.com/
http://www.dangdang.com/ http://www.m18.com/ http://www.suning.com/ http://www.hstyle.com/
)
beanstalk_jobs = [['127.0.0.1:11300'],'crawler-jobs']
user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
pm_max = 10
MultipleCrawler.new(websites, beanstalk_jobs, pm_max, user_agent).run
You appear to be running this on a Windows PC.
fork is a POSIX/Unix system call and is therefore only available on
POSIX/Unix systems.
A possible solution would be to use Cygwyn on your Windows machine.
See the Rails test guides, section 3.2 to change workers -- which use the said fork(method) to paralleled processes -- to threads. This last one use another method to parallel processes. Also take a look on the beginning of the section 3, which describes how the tests are made.
I have used a WSL2 to run Rails. However the local test needs the jruby to be executed in somewhat.
To change the parallelization method to use threads over forks put the following in your test_helper.rb
class ActiveSupport::TestCase
parallelize(workers: :number_of_processors, with: :threads)
end
After, specify in your command rails test how much workers you need, such as PARALLEL_WORKERS=1 rails test or PARALLEL_WORKERS=15 rails test.
I have a background job being run by Resque and I the process begins correctly but then I see the error:
Error
undefined method `path' for #<Hash:0x007f1900c25298>
/app/app/models/weigh_in.rb:373:in `import_without_check'
/app/app/jobs/uploads.rb:8:in `perform'
and initially this was an error but I've since updated my file and completely removed this line, pushed to heroku, restarted heroku as well as redis, yet still the error persists.
Here's the function being called in app/models/weigh_in:
def self.import_without_check(file, location_id)
error = []
success = []
options = {:key_mapping => #new_hash, :strings_as_keys => true, :keep_original_headers => true, :remove_unmapped_keys => true}
SmarterCSV.process(file, options) do |row|
hashed_row = row[0]
next if hashed_row[:scale_id].blank? || hashed_row[:scale_id].nil?
hashed_row[:unique_scale] = location_id + hashed_row[:scale_id].to_s
hashed_row = hashed_row.to_hash.except!(nil).as_json
p hashed_row
client = Client.select('id', 'name').find_by(unique_scale: hashed_row['unique_scale']) || Client.select('id', 'name').find_by(unique_mb: hashed_row['unique_scale'])
if client.nil?
error << hashed_row
next
end
hashed_row['client_id'] = client.id
program_id = client.programs.last.id
if program_id.nil? || hashed_row['client_id'].nil?
error << hashed_row
next
end
check_in = CheckIn.new(client_id: client.id, type_of_weighin: 'Standard', program_id: program_id)
unless hashed_row['date'].blank? || hashed_row['date'].nil?
p 'date', hashed_row['date']
hashed_row['date'] = Date.strptime(hashed_row["date"], "%m/%d/%y").strftime()
end
hashed_row.except!("unique_scale")
if check_in.save
hashed_row['check_in_id'] = check_in.id
end
if hashed_row['check_in_id'].nil?
error << hashed_row
next
end
weigh_in = WeighIn.new(hashed_row)
p weigh_in.valid?, weigh_in.errors.full_messages, weigh_in.errors
if weigh_in.save
success << hashed_row
end
end
return success, error
end
Is there something I need to do to make this error go away?
I solved this issue by connecting to the redis Heroku add-on via it's cli, then getting its clients, killing all connections, and restarting my heroku dynos. Once this was done, my changes were recorded:
$ heroku redis:cli
21579> client list # Returns clients
21579> client kill 10.159.107.79:43434 # use number from addr property returned with client list command
21579> quit # Exit cli
$ heroku restart
I hope this helps someone else
So I am working on some stupid simulation of mining cryptocoins - just for fun.
But I have run into an issue I want to command the client to start using a function but I am not sure how to do it, my code is bellow for Server.rb and Client.rb
Client
require 'socket'
s = TCPSocket.new 'localhost', 2626
while line = s.gets
puts line
end
s.close
Server.rb
#!/bin/ruby
require "socket"
require 'securerandom'
PORT = 2626
server_pool = TCPServer.open(PORT)
sndc_block_time = 200 # Size of block
sndc_block_time = 10 # Time to brake block 1,1 out of 200, 200
sndc_coin_blocks= 126^2 # Ammount of available blocks
sndc_coin_balance = 0
sub_blocks = 0 # Sub blocks user wants to mine
addr = SecureRandom.hex # generate a random hexadecimal address
# Notify message shown when user registers a new address
disclaimer = "Welcome to the SendCoin Network!
A new address has been registered and saved
to your computer desktop!"
def handle_connection(client)
puts "New client! #{client} \n\n"
end
puts "Listening on #{PORT}. Press CTRL+C to cancel."
while client = server_pool.accept
if Thread.new { handle_connection(client) }
if sndc_coin_blocks < 126^2 / 4
sndc_coin_blocks + 126^2 / 4 # Add a quarter of the original size
end
if Dir['../MyAddress/*.addr'].any? == true
# Statrs Mining
client.puts "Starting to mine.."
sleep(5)
client.puts "Current blocks: " + sndc_coin_blocks.to_s
client.puts "Block Size: " + sndc_coin_balance.to_s
client.puts "Approximate time to mine 1 sub-block: " + sndc_block_time.to_s
client.puts "Searching for block.."
sleep(3)
if sndc_block_time != 10
sndc_block_time = 10
end
client.puts "Started mining..."
elsif Dir['../addresses/*.addr'].any? == false
File.open("../addresses/"+addr+".addr", "w") do |f| # Create file
f.write(sndc_coin_balance.to_s)
end
client.puts "New address generated: " + addr.to_s + " , you may realunch the app now and enter your address!"
# Start Mining
puts "> File 'address.addr' has been generated for#{client}"
end
end
end
So essentially I want after this line of code:
client.puts "Started mining..."
.. to command the client to (client.rb) to start using a function.
My ruby application uses about 97 %CPU which eventually gets killed. The program is reading files from the folder and if a file name exists in the database, it skips it and checks another file. While executing this procedure, application usually gets killed.
COMMAND %CPU
ruby 96.5
Even if I insert almost all files and try to lunch an application again (because it was killed), system tends to kill it even sooner. How can I decrease the %CPU?
task :process_data, [:data_directory] => :environment do |_task, args|
# add data to a database
saver = CsvToSqlSaver.new
saver.fill_files_names
Dir.foreach(args.data_directory) do |filename|
# if not present in records already we read it
Base.logger.info "> Found #{filename}."
next if saver.files_names.to_s.include?(filename) ||
!filename.include?('csv')
Base.logger.info "> Reading #{filename}."
begin
saver.generate_db_rows_from_csv_file(args.data_directory, filename)
# handle Malformed .csv exception
rescue CSV::ArgumentError, CSV::MalformedCSVError => e
Base.logger.info e.message
next
end # we continue csv file loop?
unless integrator.insert_data_to_database
Base.logger.info '> No new data saved.'
end
end
end
This is fill_files_names:
def fill_files_names
#files_names = []
files_names = MyFilesTable.select(:filename).distinct
files_names.each do |row|
#files_names.push(row[:filename])
end
end
This is Base:
class Base
class << self
attr_accessor :logger
end
#logger ||= Logger.new(STDERR)
end
This is generate_db_rows_from_csv_file
def generate_db_rows_from_csv_file(directory, filename)
#incoming_data = []
CSV.foreach("#{directory}/#{filename}",
headers: true, quote_char: "\x00") do |csv_record|
# if invalid record, go further
next if record_invalid?(csv_record)
generate_row_in_the_database(csv_record, filename)
end
end
How do I print the require time of my files so that I can debug a slow application boot time.
I envision something like this:
$ rails s
$ app/models/user.rb took 300 milliseconds
$ ...
$ lib/connect.rb took 5000 milliseconds
Would I have to override Kernel#require and if so, how?
# ACK:
# modified from
# http://gist.github.com/264496 by github.com/eric
# http://gist.github.com/465293 by github.com/alexyoung
# USAGE:
# 1) > ruby -rrequire_tracking -e "require 'active_support'"
# 2) put in config/preinitializer.rb
# WHAT: Provides a simple overview of memory allocation occuring
# during a require.
#
# For a longer explanation, see post at:
# http://bitmonkey.net/post/308322913/tracking-initial-memory-usage-by-file-in-ruby
#
# NOTE: The numbers provided are of self + children, meaning the same will
# be attributed to multiple files at once.
#
# Also, memory that is no longer referenced (and would be freed) is
# still taken into account.
#
# It is intended to give a simple overview of allocations to track
# down gross offenders, not give an exact view of your memory usage.
require 'benchmark'
if GC.respond_to?(:enable_stats)
module RequireTracking
def require(*args)
start_allocated_size = GC.allocated_size
output = nil
benchmark = Benchmark.realtime do #Benchmark.measure
output = super
end
benchmark = (benchmark * 100000).to_i
first_caller = caller[0][40..-1].split(':')[0]
$my_require_stats << [args[0], benchmark, first_caller, (GC.allocated_size - start_allocated_size)]
end #def
end #module
else
module RequireTracking
def require(*args)
output = nil
benchmark = Benchmark.realtime do #Benchmark.measure
output = super
end
benchmark = (benchmark * 1000_00).to_i
first_caller = caller[0][40..-1].split(':')[0]
$my_require_stats << [path, benchmark, first_caller, 'NA']
end #def
end #module
puts "Warning: Not running with an interpreter with GC statistics"
end #if
module RequireTracking
$my_require_stats ||= []
$require_stats_top ||= 20
def numeric_thousands_indicators(number)
number.to_s.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2')
end
def dump_require_benchmark_stats
File.open('require_trace.log.csv', 'w') do |out|
out << "\"path\",\"benchmark\",\"caller\",\"mem\"\n"
$my_require_stats.each do |path, benchmark, caller, mem|
out << "\"#{path}\",\"#{benchmark}\",\"#{caller}\",\"#{mem}\"\n"
end
end
end
def print_require_benchmark_stats_by_memory
puts " %40s %5s %5s " % [ 'File', 'KB', 'NanoSec' ]
puts " %40s %s" % [ '-------------', '--------' ]
$my_require_stats.sort_by {|v| v[3] }.slice(0, $require_stats_top).each do |path, benchmark, caller, mem|
puts "%40s %5s %5s " % [ path, numeric_thousands_indicators(mem / 1024), benchmark ]
end
end
def print_require_benchmark_stats_by_time
puts " %40s %5s %5s " % [ 'File', 'KB', 'NanoSec' ]
puts " %40s %s" % [ '-------------', '--------' ]
$my_require_stats.sort_by {|v| v[1] }.slice(0, $require_stats_top).each do |path, benchmark, caller, mem|
puts "%40s %5s %5s " % [ path, numeric_thousands_indicators(mem / 1024), benchmark ]
end
end
end #module
Object.send(:include, RequireTracking)
Kernel.send(:include, RequireTracking)
if GC.respond_to?(:enable_stats)
GC.enable_stats
GC.clear_stats
end
at_exit do
dump_require_benchmark_stats
puts "Memory used by file:"
print_require_benchmark_stats_by_memory
puts "Time required by file:"
print_require_benchmark_stats_by_time
end