I have this rake task:
desc "get product image urls from ItemMaster"
task :get_product_image_urls => :environment do
require 'item_master'
ItemMaster.get_image
end
It calls this API method that iterates over several thousand database items like so:
class ItemMaster
include HTTParty
format :xml
base_uri 'https://api.myapi.com/v2'
def self.get_image
#items = Item.all
#items.each do |item|
response = get("/item?upc=#{item.upc}&epl=100&ef=png", :headers => {"username" => "myname", "password" => "mypass"})
image_link = response["items"]["item"]["media"]["medium"]["url"]
item_image = ItemImage.where(:upc => item.upc).first_or_create
item_image.update_attributes(:url => "#{image_link}")
end
end
end
The rake task starts up when I call it, until it hits this error about 22 items in:
undefined method `[]' for nil:NilClass
/Users/name/Rails/SG/lib/item_master.rb:12:in `block in get_image'
/Users/name/Rails/SG/lib/item_master.rb:10:in `each'
/Users/name/Rails/SG/lib/item_master.rb:10:in `get_image'
/Users/name/Rails/SG/lib/tasks/get_product_image_urls.rake:4:in `block in <top (required)>'
/Users/name/.rvm/gems/ruby-1.9.3-p429/bin/ruby_noexec_wrapper:14:in `eval'
/Users/name/.rvm/gems/ruby-1.9.3-p429/bin/ruby_noexec_wrapper:14:in `<main>'
Line 12 is this guy: image_link = response["items"]["item"]["media"]["medium"]["url"] so I'm thinking that probably a url is missing in the api and it's causing the rake task to fail. Is there a way to move past an error like this and continue on with the rest of the rake task? Thanks in advance!
The ItemMaster.get_image method will need to be edited to either not create the exception condition in the first place, or to rescue on a proper exception and move on. For more on exception handling: http://www.tutorialspoint.com/ruby/ruby_exceptions.htm
An example would be:
def self.get_image
#items = Item.all
#items.each do |item|
response = get("/item?upc=#{item.upc}&epl=100&ef=png", :headers => {"username" => "myname", "password" => "mypass"})
begin
image_link = response["items"]["item"]["media"]["medium"]["url"]
item_image = ItemImage.where(:upc => item.upc).first_or_create
item_image.update_attributes(:url => "#{image_link}")
rescue NoMethodError => ex
logger.error "Failed to locate image link ..." # Customize this to your liking
end
end
end
end
For extra goodness, consider handling the code for each item within a separate method so you can isolate responsibility for handling item-related code within the Item model itself!
Related
I'm working my first project using Neo4j. I'm parsing wikipedia's page and pagelinks dumps to create a graph where the nodes are pages and the edges are links.
I've defined some rake tasks that download the dumps, parse the data, and save it in a Neo4j database. At the end of the rake task I print the number of pages and links created, and some of the pages with the most links. Here is the output of the raks task for the zawiki.
$ rake wiki[zawiki]
[ omitted ]
...
:: Done parsing zawiki
:: 1984 pages
:: 2144 links
:: The pages with the most links are:
9625.0 - Emijrp/List_of_Wikipedians_by_number_of_edits_(bots_included): 40
1363.0 - Gvangjsih_Bouxcuengh_Swcigih: 30
9112.0 - Fuzsuih: 27
1367.0 - Cungzcoj: 26
9279.0 - Vangz_Yenfanh: 19
It looks like pages and links are being created, but when I start a rails console, or the server the links aren't found.
$ rails c
jruby-1.7.5 :013 > Pages.all.count
=> 1984
jruby-1.7.5 :003 > Pages.all.reduce(0) { |count, page| count + page.links.count}
=> 0
jruby-1.7.5 :012 > Pages.all.sort_by { |p| p.links.count }.reverse[0...5].map { |p| p.links.count }
=> [0, 0, 0, 0, 0]
Here is the rake task, and this is the projects github page. Can anyone tell me why the links aren't saved?
DUMP_DIR = Rails.root.join('lib','assets')
desc "Download wiki dumps and parse them"
task :wiki, [:wiki] => 'wiki:all'
namespace :wiki do
task :all, [:wiki] => [:get, :parse] do |t, args|
# Print info about the newly created pages and links.
link_count = 0
Pages.all.each do |page|
link_count += page.links.count
end
indent "Done parsing #{args[:wiki]}"
indent "#{Pages.count} pages"
indent "#{link_count} links"
indent "The pages with the most links are:"
Pages.all.sort_by { |a| a.links.count }.reverse[0...5].each do |page|
puts "#{page.page_id} - #{page.title}: #{page.links.count}"
end
end
desc "Download wiki page and page links database dumps to /lib/assets"
task :get, :wiki do |t, args|
indent "Downloading dumps"
sh "#{Rails.root.join('lib', "get_wiki").to_s} #{args[:wiki]}"
indent "Done"
end
desc "Parse all dumps"
task :parse, [:wiki] => 'parse:all'
namespace :parse do
task :all, [:wiki] => [:pages, :pagelinks]
desc "Read wiki page dumps from lib/assests into the database"
task :pages, [:wiki] => :environment do |t, args|
parse_dumps('page', args[:wiki]) do |obj|
page = Pages.create_from_dump(obj)
end
indent = "Created #{Pages.count} pages"
end
desc "Read wiki pagelink dumps from lib/assests into the database"
task :pagelinks, [:wiki] => :environment do |t, args|
errors = 0
parse_dumps('pagelinks', args[:wiki]) do |from_id, namespace, to_title|
from = Pages.find(:page_id => from_id)
to = Pages.find(:title => to_title)
if to.nil? || from.nil?
errors = errors.succ
else
from.links << to
from.save
end
end
end
end
end
def indent *args
print ":: "
puts args
end
def parse_dumps(dump, wiki_match, &block)
wiki_match ||= /\w+/
DUMP_DIR.entries.each do |file|
file, wiki = *(file.to_s.match(Regexp.new "(#{wiki_match})-#{dump}.sql"))
if file
indent "Parsing #{wiki} #{dump.pluralize} from #{file}"
each_value(DUMP_DIR.join(file), &block)
end
end
end
def each_value(filename)
f = File.open(filename)
num_read = 0
begin # read file until line starting with INSERT INTO
line = f.gets
end until line.match /^INSERT INTO/
begin
line = line.match(/\(.*\)[,;]/)[0] # ignore begining of line until (...) object
begin
yield line[1..-3].split(',').map { |e| e.match(/^['"].*['"]$/) ? e[1..-2] : e.to_f }
num_read = num_read.succ
line = f.gets.chomp
end while(line[0] == '(') # until next insert block, or end of file
end while line.match /^INSERT INTO/ # Until line doesn't start with (...
f.close
end
app/models/pages.rb
class Pages < Neo4j::Rails::Model
include Neo4j::NodeMixin
has_n(:links).to(Pages)
property :page_id
property :namespace, :type => Fixnum
property :title, :type => String
property :restrictions, :type => String
property :counter, :type => Fixnum
property :is_redirect, :type => Fixnum
property :is_new, :type => Fixnum
property :random, :type => Float
property :touched, :type => String
property :latest, :type => Fixnum
property :length, :type => Fixnum
property :no_title_convert, :type => Fixnum
def self.create_from_dump(obj)
# TODO: I wonder if there is a way to compine these calls
page = {}
# order of this array is important, it corresponds to the data in obj
attrs = [:page_id, :namespace, :title, :restrictions, :counter, :is_redirect,
:is_new, :random, :touched, :latest, :length, :no_title_convert]
attrs.each_index { |i| page[attrs[i]] = obj[i] }
page = Pages.create(page)
return page
end
end
I must admit that I have no idea of how Neo4j works.
Transferring from other databases though, I too assume that either some validation is wrong, or maybe even something is misconfigured in your use of the database. The latter I can't give any advice on where to look, but if it's about validation, you can look at Page#errors or try calling Page#save! and see what it raises.
One crazy idea that just came to mind looking at this example is that maybe for that relation to be configured properly, you need a back reference, too.
Maybe has_n(:links).to(Page, :links) will help you. Or, if that doesn't work:
has_n(:links_left).to(Page, :links_right)
has_n(:links_right).from(Page, :links_left)
The more I look at this, the more I think the back reference to the same table is not configured properly and thus won't validate.
Heyo,
So I built a working scraper and added the file to my app. I am now trying to take the information in the scraper and place it in my database. I am attempting to use the find_or_create method but I keep getting the following error.
breads_scraper.rb:49:in `block in summary': uninitialized constant Scraper::Bread (NameError)
from /Users/Cameron/.rvm/gems/ruby-1.9.3-p392/gems/nokogiri- 1.5.9/lib/nokogiri/xml/node_set.rb:239:in `block in each'
from /Users/Cameron/.rvm/gems/ruby-1.9.3-p392/gems/nokogiri-1.5.9/lib/nokogiri/xml/node_set.rb:238:in `upto'
from /Users/Cameron/.rvm/gems/ruby-1.9.3-p392/gems/nokogiri-1.5.9/lib/nokogiri/xml/node_set.rb:238:in `each'
from breads_scraper.rb:24:in `map'
from breads_scraper.rb:24:in `summary'
from breads_scraper.rb:57:in `<class:Scraper>'
from breads_scraper.rb:9:in `<main>'
My code looks like the following. My theory is that I am using find_or_create incorrectly, or the file doesn't know how to reach the bread method and controller.
require 'rubygems'
require 'nokogiri'
require 'open-uri'
require 'uri'
require 'json'
url = Nokogiri::HTML(open("http://en.wikipedia.org/wiki/List_of_breads"))
class Scraper
def initialize
#url = "http://en.wikipedia.org/wiki/List_of_breads"
#nodes = Nokogiri::HTML(open(#url))
end
def summary
bread_data = #nodes
breads = bread_data.css('div.mw-content-ltr table.wikitable tr')
bread_data.search('sup').remove
bread_hashes = breads.map {|x|
if content = x.css('td')[0]
name = content.text
end
if content = x.css('td a.image').map {|link| link ['href']}
image =content[0]
end
if content = x.css('td')[2]
type = content.text
end
if content = x.css('td')[3]
country = content.text
end
if content = x.css('td')[4]
description =content.text
end
{
:name => name,
:image => image,
:type => type,
:country => country,
:description => description,
}
Bread.find_or_create(:title => name, :description => description, :image_url => image, :country_origin => country, :type => type)
}
end
bready = Scraper.new
bready.summary
puts "atta boy"
end
Thanks!
Invoke the the scraper from a rake task.
lib/tasks/scraper.rake
namespace :app do
desc "Scrape breads"
task :scrape_breads => :environment do
Scraper.new.summary
end
end
Now, you can run the rake task as follows:
rake app:scrape_breads
It looks like the Bread class is not loaded.
I am using Ruby on Rails 3.1.0 and the I18n gem. I (am implementing a plugin and) I would like to check at runtime if the I18n is missing a translation key/value pairs and, if so, to use a custom string. That is, I have:
validates :link_url,
:format => {
:with => REGEX,
:message => I18n.t(
'custom_invalid_format',
:scope => 'activerecord.errors.messages'
)
}
If in the .yml file there is not the following code
activerecord:
errors:
messages:
custom_invalid_format: This is the test error message 1
I would like to use the This is the test error message 2. Is it possible? If so, how can I make that?
BTW: For performance reasons, is it advisable to check at runtime if the translation key/value pairs is present?
You could pass a :default parameter to I18n.t:
I18n.t :missing, :default => 'Not here'
# => 'Not here'
You can read more about it here.
I just had the same question and I want to compute an automatic string in case the translation is missing. If I use the :default option I have to compute the automatic string every time even when the translation is not missing. So I searched for another solution.
You can add the option :raise => true or use I18n.translate! instead of I18n.translate. If no translation can be found an exception is raised.
begin
I18n.translate!('this.key.should.be.translated', :raise => true)
rescue I18n::MissingTranslationData
do_some_resource_eating_text_generation_here
end
I don't know how to this at runtime but you can use rake to find it out. You'll have create your own rake task for that. Here's one:
namespace :i18n do
desc "Find and list translation keys that do not exist in all locales"
task :missing_keys => :environment do
def collect_keys(scope, translations)
full_keys = []
translations.to_a.each do |key, translations|
new_scope = scope.dup << key
if translations.is_a?(Hash)
full_keys += collect_keys(new_scope, translations)
else
full_keys << new_scope.join('.')
end
end
return full_keys
end
# Make sure we've loaded the translations
I18n.backend.send(:init_translations)
puts "#{I18n.available_locales.size} #{I18n.available_locales.size == 1 ? 'locale' : 'locales'} available: #{I18n.available_locales.to_sentence}"
# Get all keys from all locales
all_keys = I18n.backend.send(:translations).collect do |check_locale, translations|
collect_keys([], translations).sort
end.flatten.uniq
puts "#{all_keys.size} #{all_keys.size == 1 ? 'unique key' : 'unique keys'} found."
missing_keys = {}
all_keys.each do |key|
I18n.available_locales.each do |locale|
I18n.locale = locale
begin
result = I18n.translate(key, :raise => true)
rescue I18n::MissingInterpolationArgument
# noop
rescue I18n::MissingTranslationData
if missing_keys[key]
missing_keys[key] << locale
else
missing_keys[key] = [locale]
end
end
end
end
puts "#{missing_keys.size} #{missing_keys.size == 1 ? 'key is missing' : 'keys are missing'} from one or more locales:"
missing_keys.keys.sort.each do |key|
puts "'#{key}': Missing from #{missing_keys[key].join(', ')}"
end
end
end
put the given in a .rake file in your lib/tasks directory and execute:
rake i18n:missing_keys
Information source is here and code on github here.
If you wish to pass variable to the message like This is the test error message {variable}
This is possible using variable in language file like below.
# app/views/home/index.html.erb
<%=t 'greet_username', :user => "Bill", :message => "Goodbye" %>
# config/locales/en.yml
en:
greet_username: "%{message}, %{user}!"
More description you can find here.
I have a Rails app (2.3.8) where I send emails using ActionMailer from my Controllers, with no problems.
However, I´ve created a rake task to be called from a Cronjob (in Heroku). When those emails are sent, no locale transformations in my dates are made.
I´ve googled to find any kind of solution, but couldn´t.
Anyone can help me?
Thanks.
Here is the code:
cron.rake:
desc 'This task is called by the Heroku cron add-on'
task :cron => :environment do
puts 'Sending diary...'
hollydays = [6,0] #weekend
unless hollydays.member?(Time.zone.now.wday) #if is NOT a weekend
User.all.each do |user|
user.deliver_task_diary
end
end
puts 'done.'
end
user model method:
def deliver_task_diary
TaskMailer.deliver_task_diary(self)
end
the method in TaskMailer model:
def task_diary(user)
next_five_tasks = user.next_five_tasks
last_five_tasks = user.last_five_tasks
recipients "#{user.name} <#{user.email}>"
from "My site <no_reply#mysite.com>"
subject "Your daily tasks."
sent_on Time.zone.now
body :user => user, :next_five_tasks => next_five_tasks, :last_five_tasks => last_five_tasks
end
part of my email template that doesn´t locale:
<%=l task.estimated_delivery_date, :format => :short %>
Solved.
I don´t know if it´s the best way, but I´ve just declared the locale at the very beging of the email html template:
<% I18n.locale = "pt-BR" %>
I use tail -f to display the log file when developing my Rails app.
It shows the log messages (in color! :), which is great.
But with so much information in the 80-width console, it becomes difficult to track where a certain "set" of log messages started when, say, I clicked on a button to GET a resource.
It would be easier if there was a line number or even a time stamp at the start of each log message/line. This way I could remember that I need to start looking at the log "after line number 2365" or "after 2010/10/10 23:33:23:45".
Is this possible to do? Is there some Rails internal option for this ?
why don't you just edit your desired environment's log tags
development.rb
config.log_tags [ lambda {|r| DateTime.now } ]
If you wanted to get a time stamp:
class ApplicationController < ActionController::Base
# ...
before_filter :log_tracker
def log_tracker
Rails.logger.add(1, "Log Date: #{DateTime.now}")
end
end
And format the date however you see fit....
That would work for Rails 2.1 +, prior you could access the ActiveSupport::Buffered log object with the constant: RAILS_DEFAULT_LOGGER
Get access to the actual log file with Rails.logger.instance_values["log"]
Getting the number of lines is difficult because the logger only opens the file for writing, probably for economy. I get an IOError: not opened for reading when I try.
`
Thanks #scaney.
I found a solution here.
I modified that code to add my own coloring highlights (for development only of course!) and now I can see things like 'parameters' in yellow in the console and I'm very pleased now!
In case someone is interested, here is the code I put at the end of environment.rb.
Here is my current (dirty) implementation. Will probably fix this up later (maybe make a gem, but for now this serves me fine)
WARNING
DIRTY CODE FOLLOWS! Use at your own risk!
module ActiveSupport
class BufferedLogger
#define the ANSI escape codes for normal and bright colors
$my_my_ansi_colors = {
:normal => "\x1B[0m",
:black => "\x1B[30m",
:red => "\x1B[31m", #red
:green => "\x1B[32m",
:yellow => "\x1B[33m",
:blue => "\x1B[34m",
:magenta => "\x1B[35m",
:cyan => "\x1B[36m",
:white => "\x1B[37m",
:bred => "\x1B[1m\x1B[31m", #bright red
:bgreen => "\x1B[1m\x1B[32m",
:byellow => "\x1B[1m\x1B[33m",
:bblue => "\x1B[1m\x1B[34m",
:bmagenta => "\x1B[1m\x1B[35m",
:bcyan => "\x1B[1m\x1B[36m",
:bwhite => "\x1B[1m\x1B[37m",
}
#take a string and using the keys in the hash, replace the keys in the
#string but surround the keys with ANSI color codes
#No idea how to retain the case of the key!(TODO someday)
def my_highlight msgx,hash
return msgx if msgx.blank?
return msgx if hash.empty?
hash.each_pair do |k,v|
if not k.nil?
msgx.gsub! Regexp.new(k, Regexp::IGNORECASE), $my_my_ansi_colors[:normal]+$my_my_ansi_colors[v]+k.upcase+$my_my_ansi_colors[:normal]
end
end
msgx
end
def add(severity, message = nil, progname = nil, &block)
return if #level > severity
message = (message || (block && block.call) || progname).to_s
#INSERT BEGINS
if not $myownglobalnumbercounter.nil?
$myownglobalnumbercounter += 1
else
$myownglobalnumbercounter = 1
end
level = {
0 => "DEBUG",
1 => "INFO",
2 => "WARN",
3 => "ERROR",
4 => "FATAL"
}[severity] || "U"
message = "\x1B[0m[%d %s] : %s" % [$myownglobalnumbercounter,level,message]
message = my_highlight message, {
"debug" => :white,
"error" => :bred,
"info" => :bwhite,
"warning" => :byellow,
"warn" => :byellow ,
"parameters" => :byellow,
"#" => :bgreen,
"ms " => :bmagenta,
"GET " => :bmagenta,
"PUT " => :bmagenta,
"POST " => :bmagenta,
"DELETE " => :bmagenta
}
#INSERT ENDS
message = "#{message}\n" unless message[-1] == ?\n
buffer << message
auto_flush
message
end
end
end