Originally posted as
https://github.com/Mange/roadie-rails/issues/75
We are seeing performance issue for our daily email jobs
By using NewRelic custom instrumentation,
we found out that most time is spent in calling Roadies
Screenshot of our NewRelic data for an example worker:
The integration code:
# frozen_string_literal: true
require "rails"
require "action_controller"
require "contracts"
require "memoist"
require "roadie"
require "roadie-rails"
require "new_relic/agent/method_tracer"
module Shared::MailerMixins
module WithRoadieIntegration
# I don't want to include the constants into the class as well
module Concern
def self.included(base)
base.extend ClassMethods
end
include ::NewRelic::Agent::MethodTracer
def mail(*args, &block)
super.tap do |m|
options = roadie_options
next unless options
trace_execution_scoped(
[
[
"WithRoadieIntegration",
"Roadie::Rails::MailInliner.new(m, options).execute",
].join("/"),
],
) do
Roadie::Rails::MailInliner.new(m, options).execute
end
end
end
private
def roadie_options
::Rails.application.config.roadie.tap do |options|
options.asset_providers = [UserAssetsProvider.new]
options.external_asset_providers = [UserAssetsProvider.new]
options.keep_uninlinable_css = false
options.url_options = url_options.slice(*[
:host,
:port,
:path,
:protocol,
:scheme,
])
end
end
add_method_tracer(
:roadie_options,
"WithRoadieIntegration/roadie_options",
)
end
class UserAssetsProvider
extend(
::Memoist,
)
include(
::Contracts::Core,
::Contracts::Builtin,
)
include ::NewRelic::Agent::MethodTracer
ABSOLUTE_ASSET_PATH_REGEXP = /\A#{Regexp.escape("//")}.+#{Regexp.escape("/assets/")}/i
Contract String => Maybe[Roadie::Stylesheet]
def find_stylesheet(name)
return nil unless file_exists?(name)
Roadie::Stylesheet.new("whatever", stylesheet_content(name))
end
add_method_tracer(
:find_stylesheet,
"UserAssetsProvider/find_stylesheet",
)
Contract String => Roadie::Stylesheet
def find_stylesheet!(name)
stylesheet = find_stylesheet(name)
if stylesheet.nil?
raise Roadie::CssNotFound.new(
name,
"does not exists",
self,
)
end
stylesheet
end
add_method_tracer(
:find_stylesheet!,
"UserAssetsProvider/find_stylesheet!",
)
private
def file_exists?(name)
if assets_precompiled?
File.exists?(local_file_path(name))
else
sprockets_asset(name)
end
end
memoize :file_exists?
# If on-the-fly asset compilation is disabled, we must be precompiling assets.
def assets_precompiled?
!Rails.configuration.assets.compile
rescue
false
end
def local_file_path(name)
asset_path = asset_path(name)
if asset_path.match(ABSOLUTE_ASSET_PATH_REGEXP)
asset_path.gsub!(ABSOLUTE_ASSET_PATH_REGEXP, "assets/")
end
File.join(Rails.public_path, asset_path)
end
memoize :local_file_path
add_method_tracer(
:local_file_path,
"UserAssetsProvider/local_file_path",
)
def sprockets_asset(name)
asset_path = asset_path(name)
if asset_path.match(ABSOLUTE_ASSET_PATH_REGEXP)
asset_path.gsub!(ABSOLUTE_ASSET_PATH_REGEXP, "")
end
# Strange thing is since rails 4.2
# name is passed in like
# `/assets/mailer-a9c96bd713d0b091297b82053ccd9155b933c00a53595812d755825d1747f42d.css`
# Before any processing
# And since `sprockets_asset` is used for preview
# We just "fix" the name by removing the
#
# Regexp taken from gem `asset_sync`
# https://github.com/AssetSync/asset_sync/blob/v1.2.1/lib/asset_sync/storage.rb#L142
#
# Modified to match what we need here (we need `.css` suffix)
if asset_path =~ /-[0-9a-fA-F]{32,}\.css$/
asset_path.gsub!(/-[0-9a-fA-F]{32,}\.css$/, ".css")
end
Rails.application.assets.find_asset(asset_path)
end
add_method_tracer(
:sprockets_asset,
"UserAssetsProvider/sprockets_asset",
)
def asset_path(name)
name.gsub(%r{^[/]?assets/}, "")
end
Contract String => String
def stylesheet_content(name)
if assets_precompiled?
File.read(local_file_path(name))
else
# This will compile and return the asset
sprockets_asset(name).to_s
end.strip
end
memoize :stylesheet_content
add_method_tracer(
:stylesheet_content,
"UserAssetsProvider/stylesheet_content",
)
end
end
end
I would like to report my own findings
With NewRelic data, we think most of the time is spent on
Roadies::Inliner/selector_elements => Roadie::Inliner/elements_matching_selector
And it seems a stylesheet with more style rules will make the style inlining takes longer
Benchmark code will be something like:
# frozen_string_literal: true
require "benchmark/ips"
class TestMailer < ::ActionMailer::Base
def show(benchmark_file_path:)
return mail(
from: "somewhere#test.com",
to: ["somewhere#test.com"],
subject: "some subject",
# This is trying to workaround a strange bug in `mail` gem
# https://github.com/mikel/mail/issues/912#issuecomment-156186383
content_type: "text/html",
) do |format|
format.html do
render(
file: benchmark_file_path,
layout: false,
)
end
end
end
end
Benchmark.ips do |x|
x.warmup = 5
x.time = 60
options = Roadie::Rails::Options.new(
# Use your own provider or use built-in providers
# I use a custom provider which can be used inside a rails app,
# See https://github.com/Mange/roadie for built-in providers
#
# options.asset_providers = [UserAssetsProvider.new]
# options.external_asset_providers = [UserAssetsProvider.new]
options.keep_uninlinable_css = false
)
# Need to prepare html_file yourself with
# different stylesheet tag pointing to two different stylesheet files
x.report("fat") do
message = ::TestMailer.
show(
benchmark_file_path: "benchmark-fat-stylesheet.html",
).message.tap do |m|
Roadie::Rails::MailInliner.new(m, options).execute
end
if message.body.to_s =~ /stylesheet/
raise "stylesheet not processed"
end
end
x.report("slim") do
message = ::TestMailer.
show(
benchmark_file_path: "benchmark-slim-stylesheet.html",
).message
if message.body.to_s =~ /stylesheet/
raise "stylesheet not processed"
end
end
# Compare the iterations per second of the various reports!
x.compare!
end
For example if I have YAML file with
en:
questions:
new: 'New Question'
other:
recent: 'Recent'
old: 'Old'
This would end up as a json object like
{
'questions.new': 'New Question',
'questions.other.recent': 'Recent',
'questions.other.old': 'Old'
}
Since the question is about using YAML files for i18n on a Rails app, it's worth noting that the i18n gem provides a helper module I18n::Backend::Flatten that flattens translations exactly like this:
test.rb:
require 'yaml'
require 'json'
require 'i18n'
yaml = YAML.load <<YML
en:
questions:
new: 'New Question'
other:
recent: 'Recent'
old: 'Old'
YML
include I18n::Backend::Flatten
puts JSON.pretty_generate flatten_translations(nil, yaml, nil, false)
Output:
$ ruby test.rb
{
"en.questions.new": "New Question",
"en.questions.other.recent": "Recent",
"en.questions.other.old": "Old"
}
require 'yaml'
yml = %Q{
en:
questions:
new: 'New Question'
other:
recent: 'Recent'
old: 'Old'
}
yml = YAML.load(yml)
translations = {}
def process_hash(translations, current_key, hash)
hash.each do |new_key, value|
combined_key = [current_key, new_key].delete_if { |k| k.blank? }.join('.')
if value.is_a?(Hash)
process_hash(translations, combined_key, value)
else
translations[combined_key] = value
end
end
end
process_hash(translations, '', yml['en'])
p translations
#Ryan's recursive answer is the way to go, I just made it a little more Rubyish:
yml = YAML.load(yml)['en']
def flatten_hash(my_hash, parent=[])
my_hash.flat_map do |key, value|
case value
when Hash then flatten_hash( value, parent+[key] )
else [(parent+[key]).join('.'), value]
end
end
end
p flatten_hash(yml) #=> ["questions.new", "New Question", "questions.other.recent", "Recent", "questions.other.old", "Old"]
p Hash[*flatten_hash(yml)] #=> {"questions.new"=>"New Question", "questions.other.recent"=>"Recent", "questions.other.old"=>"Old"}
Then to get it into json format you just need to require 'json' and call the to_json method on the hash.
I am using Ruby on Rails 3.1.0 and the I18n gem. I (am implementing a plugin and) I would like to check at runtime if the I18n is missing a translation key/value pairs and, if so, to use a custom string. That is, I have:
validates :link_url,
:format => {
:with => REGEX,
:message => I18n.t(
'custom_invalid_format',
:scope => 'activerecord.errors.messages'
)
}
If in the .yml file there is not the following code
activerecord:
errors:
messages:
custom_invalid_format: This is the test error message 1
I would like to use the This is the test error message 2. Is it possible? If so, how can I make that?
BTW: For performance reasons, is it advisable to check at runtime if the translation key/value pairs is present?
You could pass a :default parameter to I18n.t:
I18n.t :missing, :default => 'Not here'
# => 'Not here'
You can read more about it here.
I just had the same question and I want to compute an automatic string in case the translation is missing. If I use the :default option I have to compute the automatic string every time even when the translation is not missing. So I searched for another solution.
You can add the option :raise => true or use I18n.translate! instead of I18n.translate. If no translation can be found an exception is raised.
begin
I18n.translate!('this.key.should.be.translated', :raise => true)
rescue I18n::MissingTranslationData
do_some_resource_eating_text_generation_here
end
I don't know how to this at runtime but you can use rake to find it out. You'll have create your own rake task for that. Here's one:
namespace :i18n do
desc "Find and list translation keys that do not exist in all locales"
task :missing_keys => :environment do
def collect_keys(scope, translations)
full_keys = []
translations.to_a.each do |key, translations|
new_scope = scope.dup << key
if translations.is_a?(Hash)
full_keys += collect_keys(new_scope, translations)
else
full_keys << new_scope.join('.')
end
end
return full_keys
end
# Make sure we've loaded the translations
I18n.backend.send(:init_translations)
puts "#{I18n.available_locales.size} #{I18n.available_locales.size == 1 ? 'locale' : 'locales'} available: #{I18n.available_locales.to_sentence}"
# Get all keys from all locales
all_keys = I18n.backend.send(:translations).collect do |check_locale, translations|
collect_keys([], translations).sort
end.flatten.uniq
puts "#{all_keys.size} #{all_keys.size == 1 ? 'unique key' : 'unique keys'} found."
missing_keys = {}
all_keys.each do |key|
I18n.available_locales.each do |locale|
I18n.locale = locale
begin
result = I18n.translate(key, :raise => true)
rescue I18n::MissingInterpolationArgument
# noop
rescue I18n::MissingTranslationData
if missing_keys[key]
missing_keys[key] << locale
else
missing_keys[key] = [locale]
end
end
end
end
puts "#{missing_keys.size} #{missing_keys.size == 1 ? 'key is missing' : 'keys are missing'} from one or more locales:"
missing_keys.keys.sort.each do |key|
puts "'#{key}': Missing from #{missing_keys[key].join(', ')}"
end
end
end
put the given in a .rake file in your lib/tasks directory and execute:
rake i18n:missing_keys
Information source is here and code on github here.
If you wish to pass variable to the message like This is the test error message {variable}
This is possible using variable in language file like below.
# app/views/home/index.html.erb
<%=t 'greet_username', :user => "Bill", :message => "Goodbye" %>
# config/locales/en.yml
en:
greet_username: "%{message}, %{user}!"
More description you can find here.
I would like my JSON output in Ruby on Rails to be "pretty" or nicely formatted.
Right now, I call to_json and my JSON is all on one line. At times this can be difficult to see if there is a problem in the JSON output stream.
Is there way to configure to make my JSON "pretty" or nicely formatted in Rails?
Use the pretty_generate() function, built into later versions of JSON. For example:
require 'json'
my_object = { :array => [1, 2, 3, { :sample => "hash"} ], :foo => "bar" }
puts JSON.pretty_generate(my_object)
Which gets you:
{
"array": [
1,
2,
3,
{
"sample": "hash"
}
],
"foo": "bar"
}
The <pre> tag in HTML, used with JSON.pretty_generate, will render the JSON pretty in your view. I was so happy when my illustrious boss showed me this:
<% if #data.present? %>
<pre><%= JSON.pretty_generate(#data) %></pre>
<% end %>
Thanks to Rack Middleware and Rails 3 you can output pretty JSON for every request without changing any controller of your app. I have written such middleware snippet and I get nicely printed JSON in browser and curl output.
class PrettyJsonResponse
def initialize(app)
#app = app
end
def call(env)
status, headers, response = #app.call(env)
if headers["Content-Type"] =~ /^application\/json/
obj = JSON.parse(response.body)
pretty_str = JSON.pretty_unparse(obj)
response = [pretty_str]
headers["Content-Length"] = pretty_str.bytesize.to_s
end
[status, headers, response]
end
end
The above code should be placed in app/middleware/pretty_json_response.rb of your Rails project.
And the final step is to register the middleware in config/environments/development.rb:
config.middleware.use PrettyJsonResponse
I don't recommend to use it in production.rb. The JSON reparsing may degrade response time and throughput of your production app. Eventually extra logic such as 'X-Pretty-Json: true' header may be introduced to trigger formatting for manual curl requests on demand.
(Tested with Rails 3.2.8-5.0.0, Ruby 1.9.3-2.2.0, Linux)
If you want to:
Prettify all outgoing JSON responses from your app automatically.
Avoid polluting Object#to_json/#as_json
Avoid parsing/re-rendering JSON using middleware (YUCK!)
Do it the RAILS WAY!
Then ... replace the ActionController::Renderer for JSON! Just add the following code to your ApplicationController:
ActionController::Renderers.add :json do |json, options|
unless json.kind_of?(String)
json = json.as_json(options) if json.respond_to?(:as_json)
json = JSON.pretty_generate(json, options)
end
if options[:callback].present?
self.content_type ||= Mime::JS
"#{options[:callback]}(#{json})"
else
self.content_type ||= Mime::JSON
json
end
end
Check out Awesome Print. Parse the JSON string into a Ruby Hash, then display it with ap like so:
require "awesome_print"
require "json"
json = '{"holy": ["nested", "json"], "batman!": {"a": 1, "b": 2}}'
ap(JSON.parse(json))
With the above, you'll see:
{
"holy" => [
[0] "nested",
[1] "json"
],
"batman!" => {
"a" => 1,
"b" => 2
}
}
Awesome Print will also add some color that Stack Overflow won't show you.
If you find that the pretty_generate option built into Ruby's JSON library is not "pretty" enough, I recommend my own NeatJSON gem for your formatting.
To use it:
gem install neatjson
and then use
JSON.neat_generate
instead of
JSON.pretty_generate
Like Ruby's pp it will keep objects and arrays on one line when they fit, but wrap to multiple as needed. For example:
{
"navigation.createroute.poi":[
{"text":"Lay in a course to the Hilton","params":{"poi":"Hilton"}},
{"text":"Take me to the airport","params":{"poi":"airport"}},
{"text":"Let's go to IHOP","params":{"poi":"IHOP"}},
{"text":"Show me how to get to The Med","params":{"poi":"The Med"}},
{"text":"Create a route to Arby's","params":{"poi":"Arby's"}},
{
"text":"Go to the Hilton by the Airport",
"params":{"poi":"Hilton","location":"Airport"}
},
{
"text":"Take me to the Fry's in Fresno",
"params":{"poi":"Fry's","location":"Fresno"}
}
],
"navigation.eta":[
{"text":"When will we get there?"},
{"text":"When will I arrive?"},
{"text":"What time will I get to the destination?"},
{"text":"What time will I reach the destination?"},
{"text":"What time will it be when I arrive?"}
]
}
It also supports a variety of formatting options to further customize your output. For example, how many spaces before/after colons? Before/after commas? Inside the brackets of arrays and objects? Do you want to sort the keys of your object? Do you want the colons to all be lined up?
Dumping an ActiveRecord object to JSON (in the Rails console):
pp User.first.as_json
# => {
"id" => 1,
"first_name" => "Polar",
"last_name" => "Bear"
}
Using <pre> HTML code and pretty_generate is good trick:
<%
require 'json'
hash = JSON[{hey: "test", num: [{one: 1, two: 2, threes: [{three: 3, tthree: 33}]}]}.to_json]
%>
<pre>
<%= JSON.pretty_generate(hash) %>
</pre>
Here is a middleware solution modified from this excellent answer by #gertas. This solution is not Rails specific--it should work with any Rack application.
The middleware technique used here, using #each, is explained at ASCIIcasts 151: Rack Middleware by Eifion Bedford.
This code goes in app/middleware/pretty_json_response.rb:
class PrettyJsonResponse
def initialize(app)
#app = app
end
def call(env)
#status, #headers, #response = #app.call(env)
[#status, #headers, self]
end
def each(&block)
#response.each do |body|
if #headers["Content-Type"] =~ /^application\/json/
body = pretty_print(body)
end
block.call(body)
end
end
private
def pretty_print(json)
obj = JSON.parse(json)
JSON.pretty_unparse(obj)
end
end
To turn it on, add this to config/environments/test.rb and config/environments/development.rb:
config.middleware.use "PrettyJsonResponse"
As #gertas warns in his version of this solution, avoid using it in production. It's somewhat slow.
Tested with Rails 4.1.6.
#At Controller
def branch
#data = Model.all
render json: JSON.pretty_generate(#data.as_json)
end
If you're looking to quickly implement this in a Rails controller action to send a JSON response:
def index
my_json = '{ "key": "value" }'
render json: JSON.pretty_generate( JSON.parse my_json )
end
Here's my solution which I derived from other posts during my own search.
This allows you to send the pp and jj output to a file as needed.
require "pp"
require "json"
class File
def pp(*objs)
objs.each {|obj|
PP.pp(obj, self)
}
objs.size <= 1 ? objs.first : objs
end
def jj(*objs)
objs.each {|obj|
obj = JSON.parse(obj.to_json)
self.puts JSON.pretty_generate(obj)
}
objs.size <= 1 ? objs.first : objs
end
end
test_object = { :name => { first: "Christopher", last: "Mullins" }, :grades => [ "English" => "B+", "Algebra" => "A+" ] }
test_json_object = JSON.parse(test_object.to_json)
File.open("log/object_dump.txt", "w") do |file|
file.pp(test_object)
end
File.open("log/json_dump.txt", "w") do |file|
file.jj(test_json_object)
end
I have used the gem CodeRay and it works pretty well. The format includes colors and it recognises a lot of different formats.
I have used it on a gem that can be used for debugging rails APIs and it works pretty well.
By the way, the gem is named 'api_explorer' (http://www.github.com/toptierlabs/api_explorer)
if you want to handle active_record object, puts is enough.
for example:
without puts
2.6.0 (main):0 > User.first.to_json
User Load (0.4ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT $1 [["LIMIT", 1]]
=> "{\"id\":1,\"admin\":true,\"email\":\"admin#gmail.com\",\"password_digest\":\"$2a$10$TQy3P7NT8KrdCzliNUsZzuhmo40LGKoth2hwD3OI.kD0lYiIEwB1y\",\"created_at\":\"2021-07-20T08:34:19.350Z\",\"updated_at\":\"2021-07-20T08:34:19.350Z\",\"name\":\"Arden Stark\"}"
with puts
2.6.0 (main):0 > puts User.first.to_json
User Load (0.3ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT $1 [["LIMIT", 1]]
{"id":1,"admin":true,"email":"admin#gmail.com","password_digest":"$2a$10$TQy3P7NT8KrdCzliNUsZzuhmo40LGKoth2hwD3OI.kD0lYiIEwB1y","created_at":"2021-07-20T08:34:19.350Z","updated_at":"2021-07-20T08:34:19.350Z","name":"Arden Stark"}
=> nil
if you are handle the json data, JSON.pretty_generate is a good alternative
Example:
obj = {foo: [:bar, :baz], bat: {bam: 0, bad: 1}}
json = JSON.pretty_generate(obj)
puts json
Output:
{
"foo": [
"bar",
"baz"
],
"bat": {
"bam": 0,
"bad": 1
}
}
if it's in the ROR project, I always prefer to use gem pry-rails to format my codes in the rails console rather than awesome_print which is too verbose.
Example of pry-rails:
it also has syntax highlight.
# example of use:
a_hash = {user_info: {type: "query_service", e_mail: "my#email.com", phone: "+79876543322"}, cars_makers: ["bmw", "mitsubishi"], car_models: [bmw: {model: "1er", year_mfc: 2006}, mitsubishi: {model: "pajero", year_mfc: 1997}]}
pretty_html = a_hash.pretty_html
# include this module to your libs:
module MyPrettyPrint
def pretty_html indent = 0
result = ""
if self.class == Hash
self.each do |key, value|
result += "#{key}: #{[Array, Hash].include?(value.class) ? value.pretty_html(indent+1) : value}"
end
elsif self.class == Array
result = "[#{self.join(', ')}]"
end
"#{result}"
end
end
class Hash
include MyPrettyPrint
end
class Array
include MyPrettyPrint
end
Simplest example, I could think of:
my_json = '{ "name":"John", "age":30, "car":null }'
puts JSON.pretty_generate(JSON.parse(my_json))
Rails console example:
core dev 1555:0> my_json = '{ "name":"John", "age":30, "car":null }'
=> "{ \"name\":\"John\", \"age\":30, \"car\":null }"
core dev 1556:0> puts JSON.pretty_generate(JSON.parse(my_json))
{
"name": "John",
"age": 30,
"car": null
}
=> nil
Pretty print variant (Rails):
my_obj = {
'array' => [1, 2, 3, { "sample" => "hash"}, 44455, 677778, nil ],
foo: "bar", rrr: {"pid": 63, "state with nil and \"nil\"": false},
wwww: 'w' * 74
}
require 'pp'
puts my_obj.as_json.pretty_inspect.
gsub('=>', ': ').
gsub(/"(?:[^"\\]|\\.)*"|\bnil\b/) {|m| m == 'nil' ? 'null' : m }.
gsub(/\s+$/, "")
Result:
{"array": [1, 2, 3, {"sample": "hash"}, 44455, 677778, null],
"foo": "bar",
"rrr": {"pid": 63, "state with nil and \"nil\"": false},
"wwww":
"wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww"}
If you are using RABL you can configure it as described here to use JSON.pretty_generate:
class PrettyJson
def self.dump(object)
JSON.pretty_generate(object, {:indent => " "})
end
end
Rabl.configure do |config|
...
config.json_engine = PrettyJson if Rails.env.development?
...
end
A problem with using JSON.pretty_generate is that JSON schema validators will no longer be happy with your datetime strings. You can fix those in your config/initializers/rabl_config.rb with:
ActiveSupport::TimeWithZone.class_eval do
alias_method :orig_to_s, :to_s
def to_s(format = :default)
format == :default ? iso8601 : orig_to_s(format)
end
end
I use the following as I find the headers, status and JSON output useful as
a set. The call routine is broken out on recommendation from a railscasts presentation at: http://railscasts.com/episodes/151-rack-middleware?autoplay=true
class LogJson
def initialize(app)
#app = app
end
def call(env)
dup._call(env)
end
def _call(env)
#status, #headers, #response = #app.call(env)
[#status, #headers, self]
end
def each(&block)
if #headers["Content-Type"] =~ /^application\/json/
obj = JSON.parse(#response.body)
pretty_str = JSON.pretty_unparse(obj)
#headers["Content-Length"] = Rack::Utils.bytesize(pretty_str).to_s
Rails.logger.info ("HTTP Headers: #{ #headers } ")
Rails.logger.info ("HTTP Status: #{ #status } ")
Rails.logger.info ("JSON Response: #{ pretty_str} ")
end
#response.each(&block)
end
end
I had a JSON object in the rails console, and wanted to display it nicely in the console (as opposed to displaying like a massive concatenated string), it was as simple as:
data.as_json