Using Rails 5 and Rspec 3.7 I have a fairly simple test that is currently flapping (sometimes passing sometimes failing). Through the debugging I've done so far it seems that values i'm saving to my test database are not persisting between tests, but I can not figure out why this is the case.
Here is the test suite with comments on the flapping test (the rest consistently pass)
describe ResourceCenterController, type: :controller do
before(:each) do
#platform_instance = FactoryBot.create(:platform_instance)
#domain = FactoryBot.create(:domain, platform_instance: #platform_instance)
#user = FactoryBot.create(:user, platform_instance: #platform_instance, first_name: "O'flaggan")
end
context 'when user IS signed in' do
before(:each) do
login_user(#user)
end
context 'when user in ONE community' do
before(:each) do
#user.communities = [#platform_instance.communities.first]
#user.save!
end
describe '#index' do
before(:each) do
#rc = FactoryBot.create(:resource_center, platform_instance: #platform_instance, launch_at: nil, expire_at: nil)
end
context 'when community assigned NO resource centers' do
before(:each) do
#rc.communities = []
#rc.save!
get :index
end
it_behaves_like '200 w name in body' do
let(:names) { ['There are no files for your review at the moment.'] }
end
end
context 'when community assigned ONE resource center' do
before(:each) do
#rc.communities = [#user.communities.first]
#rc.save!
end
context 'when resource center assigned NO mediafiles' do
before(:each) do
#rc.mediafiles = []
#rc.save!
get :index
end
it_behaves_like '200 w name in body' do
let(:names) { ['There are no files for your review at the moment.'] }
end
end
# this test is flapping
# sometimes it will persist the mediafile and it will show up
# other times it will be saved, why is that?
context 'when resource center assigned ONE mediafile' do
before(:each) do
#mediafile = FactoryBot.create(:mediafile, platform_instance: #platform_instance)
#rc.mediafiles << #mediafile
#rc.save!
get :index
end
it_behaves_like '200 w name in body' do
let(:names) { ["#{#mediafile.name}"] }
end
end
end
end
end
end
end
Here is the shared context
shared_context '200 w name in body' do
it 'returns 200' do
expect(response.status).to eq(200)
end
it 'renders the view' do
names.each do |name|
expect(response.body).to include(name)
end
end
end
Edit: I learned of the bisect flag and ran it with this output
Bisect started using options: "spec/controllers/resource_center_controller_spec.rb"
Running suite to find failures... (7.39 seconds)
Starting bisect with 1 failing example and 5 non-failing examples.
Checking that failure(s) are order-dependent... failure appears to be order-dependent
Round 1: bisecting over non-failing examples 1-5 .. multiple culprits detected - splitting candidates (13.84 seconds)
Round 2: bisecting over non-failing examples 1-3 . ignoring examples 1-2 (6.95 seconds)
Round 3: bisecting over non-failing examples 4-5 . ignoring example 4 (6.75 seconds)
Bisect complete! Reduced necessary non-failing examples from 5 to 2 in 34.1 seconds.
The minimal reproduction command is:
rspec ./spec/controllers/resource_center_controller_spec.rb[1:1:1:1:2:1:1:1,1:1:1:1:2:2:1:1,1:1:1:1:2:2:1:2]
Edit: here is the factory for mediafile
FactoryBot.define do
# pi = PlatformInstance.select
factory :mediafile do
name { Faker::Simpsons.character }
platform_instance_uuid { PlatformInstance.first.uuid } # stick to platforminstance.first for now
platform_instance { PlatformInstance.first } # had tried to use a variable, but was
# not working
description { Faker::Simpsons.quote }
document { File.new("#{Rails.root}/spec/support/fixtures/mediafiles/document_01.pdf") }
image { File.new("#{Rails.root}/spec/support/fixtures/mediafiles/image_01.jpg") }
# review_with_mediafiles will create mediafile data after the review has been created
factory :mediafile_with_review do
after(:create) do |mediafile, evaluator|
create(:review, mediafile: mediafile)
end
end
end
end
and here is the factory for resource center
FactoryBot.define do
factory :resource_center do
title { Faker::Company.catch_phrase }
description { Faker::Lorem.paragraph(10) }
launch_at { Time.now }
expire_at { Time.now + 1.week }
platform_instance_uuid { PlatformInstance.first.uuid } # stick to PlatformInstance.first for now
platform_instance { PlatformInstance.first } # had tried to use a variable, but was
# not working
status { [:testing, :live].sample }
# review_with_mediafiles will create mediafile data after the review has been created
# this factory inherits everything from the factory it is nested under
factory :resource_center_with_mediafiles do
after(:create) do |resource_center, evaluator|
create(:mediafile, resource_centers: [resource_center])
end
end
end
end
The controller method itself is fairly simple
def index
#resource_centers = current_user.resource_centers.within_dates
end
current_user variable is assigned in the application controller which I don't think is super necessary to include here. The view is also fairly simple and can be seen below
-content_for :breadcrumbs do
=render 'layouts/shared/breadcrumbs', breadcrumbs: [link_to('Home', user_root_path), 'Resource Center']
-files_present = false
-#resource_centers.each do |resource_center|
-if resource_center.mediafiles.present?
-files_present = true
%h3.color-primary= resource_center.title.html_safe
=resource_center.description.html_safe
.space-above-2
-resource_center.mediafiles.sort.each do |mediafile|
=render 'resource_center/mediafile_item', resource_center: resource_center, mediafile: mediafile
-if !files_present
%h4 There are no files for your review at the moment.
Here is the partial rendered in the above view.
.index-list
.index-item.large-avatar
.item-avatar
=link_to resource_center_mediafile_view_path(resource_center, mediafile) do
= image_tag mediafile.image.url
.item-content
.item-header= mediafile.name
.item-attribute-list
%span.item-attribute
-if mediafile.duration.present?
%strong DURATION:
=pluralize(mediafile.duration, "minute")
-if mediafile.document.size.to_i > 0
%strong SIZE:
=number_to_human_size(mediafile.document.size)
.item-actions
-if resource_center.downloadable
=link_to 'Download', mediafile.download_url, class: 'mui-button default', target: '_blank'
=link_to 'View', resource_center_mediafile_view_path(resource_center, mediafile), class: 'mui-button'
Here is the spec_helper file:
ENV['RAILS_ENV'] ||= 'test'
require File.expand_path("../../config/environment", __FILE__)
require 'rspec/rails'
RSpec.configure do |config|
config.before(:each) do
#only modify the request when testing controllers
if described_class <= ApplicationController
request.host = 'localhost:3000'
end
end
config.include Rails.application.routes.url_helpers
config.expect_with :rspec do |expectations|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
end
config.mock_with :rspec do |mocks|
mocks.verify_partial_doubles = true
end
config.before(:suite) do
DatabaseCleaner.strategy = :transaction
DatabaseCleaner.clean_with(:truncation)
end
config.around(:each) do |example|
DatabaseCleaner.cleaning do
example.run
end
end
config.before(:all) do
DatabaseCleaner.start
end
config.after(:all) do
DatabaseCleaner.clean
end
config.shared_context_metadata_behavior = :apply_to_host_groups
# config.include Rails.application.routes.url_helpers
end
Please let me know if there is other information that would be helpful. I think this is something wrong with my test suite, particularly the before(:each) blocks, but my experimentation has not given me any insights.
Disclaimer: I did not read the whole code you posted, so I won't give you the answer what causes this flakiness (or flappines as you call it) but I'll give you a method to find it yourself. (fish vs. fishing rod thing kinda thing)
Using bisect is great, and since it says that the issue is order dependent it's fairly easy to continue.
You can now set a breakpoint in the failing it and investigate why the results are different than expected. Most probably there's some leftover junk in the DB left from some other spec.
When you pinpoint the reason for the failing spec, you can run command:
rspec --format doc \
./spec/controllers/resource_center_controller_spec.rb[1:1:1:1:2:1:1:1,1:1:1:1:2:2:1:1,1:1:1:1:2:2:1:2]
This will tell you in what order the tests are run (since [1:1:1:1:2:1:1:1,1:1:1:1:2:2:1:1,1:1:1:1:2:2:1:2] is not very human friendly)
and you can look for the spec that leaves the "state unclean" (mentioned DB junk, but could be something else)
When you pin-point the offender you can add some crude fix (like Model.destroy_all after it, to confirm that it's The Reason).
Please note that this is not the proper fix yet.
After you confirm that this is true - you're ready to search for a solution. This can be using DBCleaner for your specs, or fixing some cache code that is misbehaving or something completely different (feel free to ask another question when you have the answers)
One extra note: in many projects order of the specs will be randomized. In such case bisecting will fail unless you know the --seed under which the specs fail.
Related
I have been trying to test a rake task with RSPEC. So far my understanding of the problem is that DB transactions are creating different contexts for each aspects of the test (one in the test context and one in the rake task execution context). Here are the two files I'm using in my test:
One to make rake tasks available in Rspec and be able to call task.execute:
...spec/support/tasks.rb
require "rake"
module TaskExampleGroup
extend ActiveSupport::Concern
included do
let(:task_name) { self.class.top_level_description.sub(/\Arake /, "") }
let(:tasks) { Rake::Task }
subject(:task) { tasks[task_name] }
end
end
RSpec.configure do |config|
config.define_derived_metadata(:file_path => %r{/spec/tasks/}) do |metadata|
metadata[:type] = :task
end
config.include TaskExampleGroup, type: :task
config.before(:suite) do
Rails.application.load_tasks
end
end
One to properly test my rake task:
...spec/tasks/reengage_spec.rb
require "rails_helper"
describe "rake reengage:unverified_users", type: :task do
let(:confirmed_user){create(:user, :confirmed)}
it "Preloads the Rails environment" do
expect(task.prerequisites).to include "environment"
end
it "Runs gracefully with no users" do
expect { task.execute }.not_to raise_error
end
it "Logs to stdout without errors" do
expect { task.execute }.to_not output.to_stderr
end
it "Reengage confirmed users" do
task.execute
expect(confirmed_user).to have_attributes(:reactivated_at => DateTime.now)
end
end
So far, when the rake task is executed through Rspec, it seems like the user I have created is not available in the context of rake, hence his . reactivated_at remains nil. I though I could use database cleaner to be able to configure DB transactions and preserve the context but I have no idea where to start.
I could end putting all this code in a lib and test it separately but I'm wondering if there is a way to do this. Is there a straightforward way to share the context of execution or to make my FactoryGirl user available when the task is initialized?
I'm trying to write some tests involving file operations. I want to use some fake file system (something like VCR for external services) and I have found fakeFS. Unfortunately, either I can't set it right or something is broken (which I doubt, it's quite basic function), I've prepared simple example which illustrates what I mean, let the code speak:
With real FS:
module MyModule
describe Something do
before(:all) do
File.open("#{Rails.root}/foo.txt", 'w+') { |f| f.write 'content'}
end
it 'should exist' do
expect(Pathname.new("#{Rails.root}/foo.txt").exist?).to be_true
end
it 'should still exist' do
expect(Pathname.new("#{Rails.root}/foo.txt").exist?).to be_true
end
end
end
Running that gives:
bash-4.2$ rspec
..
Finished in 0.00161 seconds
2 examples, 0 failures
Adding fakeFS in such way:
require 'fakefs/spec_helpers'
module MyModule
describe Something do
include FakeFS::SpecHelpers
FakeFS.activate!
FakeFS::FileSystem.clone(Rails.root)
before(:all) do
File.open("#{Rails.root}/foo.txt", 'w+') { |f| f.write 'content'}
end
it 'should exist' do
expect(Pathname.new("#{Rails.root}/foo.txt").exist?).to be_true
end
it 'should still exist' do
expect(Pathname.new("#{Rails.root}/foo.txt").exist?).to be_true
end
end
end
results in:
bash-4.2$ rspec
.F
Failures:
1) MyModule::Something should still exist
Failure/Error: expect(Pathname.new("#{Rails.root}/foo.txt").exist?).to be_true
expected: true value
got: false
# ./spec/models/something_spec.rb:23:in `block (2 levels) in <module:MyModule>'
Finished in 0.00354 seconds
2 examples, 1 failure
So it seems like file is not persisted through subsequent tests. Do I misunderstand how before(:all) works or do I do something wrong? If so then why that code works with real files?
If it is 'not a bug, just a feature' then is there any other fake filesystem gem which is consistent with real one? Or do I have to stay with real files to get tests that.. well, test?
I found the answer just after creating that question, duh ;) I've looked into source of that lib and found suspicious line.
Instead of FakeFS::SpecHelpers I've included FakeFS::SpecHelpers::All which is the same code except FakeFS::FileSystem is not being cleared after each call, now it behaves correctly.
Whenever I run a user test, RSpec leaves the Fabricated user in the test database after the test has completed, which is messing up my other tests. I will do a rake db:test:prepare, but when I run my tests again, the record is recreated in my database. I have no idea why this is happening. It only happens with user objects.
In my spec_helper file I even have:
config.use_transactional_fixtures = true
Here is an example test that creates a record:
it "creates a password reset token for the user" do
alice = Fabricate(:user)
post :create, email: alice.email
expect(assigns(alice.password_reset_token)).to_not eq(nil)
end
Fabricator:
Fabricator(:user) do
email { Faker::Internet.email }
password 'password'
name { Faker::Name.name }
end
Could this have anything to do with my users model?
you should use a gem called database_cleaner that will truncate your database and reset everything automatically so in your gem file add the gem database_cleaner after that inside your spec_helper.rb configure it
spec_helper.rb
config.use_transactional_fixtures = false
config.before(:suite) do
DatabaseCleaner.strategy = :truncation
end
config.before(:each) do
DatabaseCleaner.start
end
config.after(:each) do
DatabaseCleaner.clean
end
and then create a new file in your spec/support directory
spec/support/shared_db_connection.rb
class ActiveRecord::Base
mattr_accessor :shared_connection
##shared_connection = nil
def self.connection
##shared_connection || retrieve_connection
end
end
ActiveRecord::Base.shared_connection=ActiveRecord::Base.connection
Now whenever you run your tests the database will be reset.This was taken from the book 'Everyday Rails testing with RSpec' by Aaron Sumner
Each test is wrapped in a database transaction. That means that everything created during the test should be gone when the test finishes. Therefore, I would suspect that whatever you have in your database was made outside the test itself (like in a before(:all) block).
Also this doesn't guarantee that your database will be empty each time you run your tests. It might be possible that you accidentally added a record somehow, and now it just keeps reverting to that state.
If you want to make sure your tests have a shiny database each time, you should have a look at the database_cleaner gem.
The simplest solution is to make sure RSpec tests run in transactions (Rails does this by default)
spec_helper.rb
config.around(:each) do |example|
ActiveRecord::Base.transaction do
example.run
raise ActiveRecord::Rollback
end
end
If I had to guess, the line post :create, email: alice.email seems like a likely candidate for doing the actual user creation.
Stub that line out with an bogus test and see if you're still getting a user created in the DB.
My problem turned out to be that I was was using before(:all) in my RSpec files which I found out the hard way that the data created does not get rolled back. I switched to before(:example) per this article: https://relishapp.com/rspec/rspec-rails/docs/transactions
database_cleaner works for the most part but on some things where i am expecting something like user_id to clear after each test it doesn't. so, the user_id will increment throughout instead of clearing and the user id is predictable as 1, 2, 3 or however many that are created for the test. i can just call the id instead of hardcoding the expected result but later on i really need it to clear that stuff in more complex examples. this is the easiest for showing. any help would be greatly appreciated.
FROM SPEC_HELPER.RB:
RSpec.configure do |config|
config.mock_with :rspec
config.include FactoryGirl::Syntax::Methods
config.include(Capybara, :type => :integration)
config.include Devise::TestHelpers, :type => :controller
config.use_transactional_fixtures = false
config.before(:each) do
I18n.default_locale = :en
I18n.locale = :en
DatabaseCleaner.start
ResqueSpec.reset!
ActionMailer::Base.deliveries.clear
end
config.after(:each) do
DatabaseCleaner.clean
end
config.after(:all) do
TestCleaner.clean
end
config.before(:suite) do
DatabaseCleaner.clean_with(:truncation)
DatabaseCleaner.strategy = :transaction
Role.reset_cache!
end
config.after(:suite) do
DatabaseCleaner.clean_with(:truncation)
end
FROM MY TEST:
it "should return one provider" do
get :index
response.body.gsub(/\s+/, "").should == {
:experts => [{
:availability => false,
:name => "#{#provider.user.first_name}#{#provider.user.last_name}",
:expert_id => 1,
:photo => #provider.details.photo.url
}] }.to_json
end
it "should show return two providers" do
#provider2 = create(:provider)
get :index
response.body.gsub(/\s+/, "").should == {
:experts => [{
:availability => false,
:name => "#{#provider.user.first_name}#{#provider.user.last_name}",
:expert_id => 1,
:photo => #provider.details.photo.url
},
{
:availability => false,
:name => "#{#provider.user.first_name}#{#provider.user.last_name}",
:expert_id => 2,
:photo => #provider.details.photo.url
}
] }.to_json
end
Database cleaner is wrapping each of your specs in a transaction, and rolling back that transaction at the end of the spec to remove any database changes. Rolling back a transaction doesn't reset the autoincrement or sequence values used to auto assign the primary key though.
I'd strongly recommend not hardcoding the ids. You mention that your examples will get more complicated, in which case having random integers sprinkled through your code will be even less maintainable than in simpler examples. Assuming you're using mysql then using the truncation strategy will reset autoincrement values, but it is also a lot slower.
Is it the request/integration specs where the database is not cleaning properly? If so it's possibly because you're cleaning with a transaction strategy and not a truncation strategy.
Try adding this, which changes the strategy to truncation for integration specs only:
config.before type: :integration do
DatabaseCleaner.strategy = :truncation
end
You can't use a transaction strategy for integration specs because Capybara runs in a separate thread with a different database connection.
This article helped me a lot in setting up my DB stuff for rspec/capybara: Sane Rspec config for clean, and slightly faster, specs.
I ran into this a lot when starting testing. I eventually found that it was not due to the database cleaner issue (as I also suspected), but rather the structure of the code that does the tests.
The best way I can try and descibe is is to say that basically if you do things outside of your before(:each) set up blocks and the actual it and should it end up being 'outside' of the actual test and causes these issues.
Specifically I suspect that there could be a problem here:
it "should show return two providers" do
#provider2 = create(:provider)
get :index
response.body.gsub(/\s+/, "").should == {
I would look to change this to something more like:
it "should show return two providers" do
before(:each) do {
#provider2 = create(:provider)
get :index
}
response.body.gsub(/\s+/, "").should == {
My environment:
jruby-1.5.3
Rails 2.3.8
RSpec 1.3.1
Windows 7 (64-bit machine)
Running Rspec with the following source code, why does rspec read line marked with '=>' which is context before the statement before(:each). Any help much appreciated
def save_env
#host_os = Config::CONFIG['host_os']
end
def restore_env
Config::CONFIG['host_os'] = #host_os
end
describe Manager::ManagerConfig do
before(:each) do
save_env
end
after(:each) do
restore_env
end
context "Within Linus/Solaris environment" do
=> Config::CONFIG['host_os'] = 'linux'
it "should return the correct manager path under linux/solaris" do
# bar
end
it "should return the correct license path under windows env" do
# foo
end
end
end
A context sets up an inner class, so the lines within it are going to be executed at load time, except that each it, before and after creates a block of code that will be executed later.
All you need to do is wrap the config setup in its own before(:each) block, and the order will be what you expect: The outer before(:each), then the inner before(:each), then the it:
before(:each) do
Config::CONFIG['host_os'] = 'linux'
end