I have a bunch of apps uploaded to the Testflight. How to automatically download all the builds to my computer?
You can use Ruby + Mechanize (https://github.com/sparklemotion/mechanize):
require 'mechanize'
#agent = Mechanize.new
def process_login_page page
puts 'Login...'
login_form = page.forms.first # form has no action or name so just take the first one
login_field = login_form.field_with(:name => 'username')
password_field = login_form.field_with(:name => 'password')
login_field.value = 'username'
password_field.value = 'password'
login_form.submit
app_link_pattern = /\/dashboard\/applications\/(.*?)\/token\//
puts 'Dashboard...'
#agent.get("https://testflightapp.com/dashboard/applications/") do |dashboard_page|
dashboard_page.links.each do |link|
link.href =~ app_link_pattern
if $1 != nil
puts "Builds page for #{$1}..."
#agent.get "https://testflightapp.com/dashboard/applications/#{$1}/builds/" do |builds_page|
process_builds_page builds_page
end
end
end
end
end
def process_builds_page page
body = page.body
build_pages = body.scan /<tr class="goversion pointer" id="\/dashboard\/builds\/report\/(.*?)\/">/
build_pages.each do |build_id|
#agent.get "https://testflightapp.com/dashboard/builds/complete/#{build_id.first}/" do |build_page|
process_build_page build_page
end
end
end
def process_build_page page
build_link = page.links_with(:dom_class => 'bitly').first
#agent.get("https://www.testflightapp.com#{build_link.href}") { |install_page| process_install_page install_page}
end
def process_install_page page
# we need to figure out what kind of build is that
ipa_link = page.link_with(:text => "download the IPA.")
if (ipa_link != nil)
download_build ipa_link, "ipa"
else
apk_link = page.link_with(:text => "download the APK.")
if (apk_link != nil)
download_build apk_link, "apk"
end
end
end
def download_build link, file_ext
link.href =~ /\/dashboard\/ipa\/(.*?)\//
filename = "#{$1}.#{file_ext}"
file_url = "https://www.testflightapp.com#{link.href}"
puts "Downloading #{file_url}..."
#agent.get(file_url).save("out/#{filename}")
end
FileUtils.rm_rf "out"
Dir.mkdir "out"
login_page_url = "https://testflightapp.com/login/"
#agent.get(login_page_url) { |page| process_login_page page }
Disclaimer: I'm not a Ruby developer and this code is way far from being well designed or safe. Just a quick and dirty solution.
Related
so im trying to do web scraping with rails and kimurai, the problem i ran in to was that for some reason i get a single big object instead of one for each of the products im scraping, here is my code:
class ProductsSpider < Kimurai::Base
#name = "products_spider"
#engine = :mechanize
def self.process(url)
#start_urls = [url]
self.crawl!
end
def parse(response, url:, data: {})
response.xpath("//div[#class='andes-card andes-card--flat andes-card--default ui-search-result ui-search-result--core andes-card--padding-default andes-card--animated']").each do |product|
item = {}
item[:product_name] = product.xpath("//h2[#class='ui-search-item__title ui-search-item__group__element']")&.text&.squish
item[:price] = product.xpath("//span[#class='price-tag-fraction']")&.text&.squish&.delete('^0-9')to_i
item[:shipping] = product.xpath("//p[#class='ui-search-item__shipping ui-search-item__shipping--free']")&.text&.squish
Product.where(item).first_or_create
end
end
end
and here is the function on the controller:
def scrape
url = "https://computacion.mercadolibre.com.ar/componentes-pc-placas-video/msi/cordoba/placa-de-video_NoIndex_True#applied_filter_id%3Dstate%26applied_filter_name%3DUbicaci%C3%B3n%26applied_filter_order%3D13%26applied_value_id%3DTUxBUENPUmFkZGIw%26applied_value_name%3DC%C3%B3rdoba%26applied_value_order%3D11%26applied_value_results%3D120%26is_custom%3Dfalse%26view_more_flag%3Dtrue"
response = ProductsSpider.process(url)
if response[:status] == :completed && response[:error].nil?
flash.now[:notice] = "Successfully scraped url"
else
flash.now[:alert] = response[:error]
end
rescue StandardError => e
flash.now[:alert] = "Error: #{e}"
end
I'm using Rails 4.2, sidekiq, rubyzip, axlsx and axlsx-rails
What I need to do:
I need to export file in a mail, but as a link and not as attachment.
I want an email send with a url to the download for it
Problem:
I don't know how to do it with this gem, in the git documentation there's nothing about it and every url I use don't work. I need to know how to create a download link for it and not as attachment to the mail
Code:
controller:
def export
export_args = {account_id: current_account.id}
ExportReportJob.perform_later(export_args)
redirect_to action: :index
end
job:
def perform(export_args)
begin
export_failed = false
account = Admin::Account.find_by(id: export_args[:account_id])
account.connect_to_target_db
#accounts = Admin::Account.active_accounts
file_name = "#{Time.now.strftime("%d_%m_%Y_at_%I_%M%p")}_export.xlsx"
dir_path = "#{Rails.root}/public/exports"
FileUtils.mkdir_p(dir_path) unless File.directory?(dir_path)
file_location = "#{absolute_path}/#{file_name}"
Admin::Mailer.export(account.pi, #accounts, file_location).deliver
rescue => e
export_failed = true
Airbrake.notify(e, environment_name: Rails.env, error_message: "export account failed #{e.message}")
ensure
ActiveRecord::Base.clear_active_connections!
end
end
template:
wb = xlsx_package.workbook
wb.add_worksheet(name: "Accounts") do |sheet|
sheet.add_row ["Account id", "Account name", "Organization"]
accounts.each do |account|
sheet.add_row [account.id, accoount.name, account.organization_id]
end
end
xlsx_package.to_stream.read
mailer:
def export(member, accounts, file_location)
#member = member
#title = "Export"
#header = subject = "Your Export is Ready"
#link = file_location
xlsx = render_to_string(layout: false, handlers: [:xlsx], template: "export.xlsx.axlsx", locals: {accounts: accounts})
# attachment = Base64.encode64(xlsx)
# attachments["export_#{DateTime.now.strftime("%d_%m_%Y_%HH%MM")}.xlsx"] = {mime_type: Mime::XLSX, content: attachment, encoding: 'base64'}
# ^ this is not what I want, I want to get the url for it and insert to #link
mail(:to => member.email, :subject => subject) do |format|
format.html
end
end
mail template:
= render partial: 'header'
= h_tag 'The export you requested has been completed'
= button 'Download the file', #link, { download: true }
= p_tag '(the file download will expire after <b>2</b> days)'
= render partial: 'footer'
Here is a simple case of spec in Rails 4.2:
it "returns redirect to Save & Copy" do
user_access = FactoryGirl.create(:user_access, :action => 'create', :resource =>'simple_orderx_orders', :role_definition_id => #role.id, :rank => 1,
:sql_code => "")
session[:user_id] = #u.id
q1 = FactoryGirl.create(:simple_orderx_order)
q = FactoryGirl.attributes_for(:simple_orderx_order)
FactoryGirl.create(:project_misc_definitionx_misc_definition, :resource_id => q1.id, :resource_string => 'simple_orderx_orders', definition_category: 'production_step')
expect {
get 'create', {:order => q, commit: I18n.t('Save & Copy')}
expect(response).to redirect_to URI.escape(SUBURI + "/view_handler?index=0&msg=Successfully Saved!")
}.to change(ProjectMiscDefinitionx::MiscDefinition.all.reload, :count).by(1)
end
end
In debugger, there is a new record being added to table MiscDefinition and MiscDefinition.all.count is 2, but in spec the ProjectMiscDefinitinox::MiscDefinition.all.count is always 0 and the spec case fails. What could cause the counting error in spec?
Here is the create in controller:
def create
#order = SimpleOrderx::Order.new(new_params)
#order.last_updated_by_id = session[:user_id]
#order.entered_by_id = session[:user_id]
#order.fort_token = session[:fort_token]
copy_steps(#order) if params[:commit] == I18n.t('Save & Copy') #copy mfg steps from previous order
if #order.save
copy_steps(#order.reload) if params[:commit] == I18n.t('Save & Copy') #copy mfg steps from previous order
redirect_to URI.escape(SUBURI + "/view_handler?index=0&msg=Successfully Saved!")
else
#erb_code = find_config_const('order_new_view', session[:fort_token], 'simple_orderx')
flash[:notice] = t('Data Error. Not Saved!')
render 'new'
end
end
def copy_steps(order)
obj = Order.where('drawing_num = ? AND id != ?', order.drawing_num, order.id).order('created_at DESC') if order.drawing_num.present? #find the previous order of the same drawing#
obj = Order.where('part_num = ? AND id != ?', order.part_num, order.id).order('created_at DESC') if obj.blank? && order.part_num.present?
obj = Order.where('part_name Like ? AND id != ?', "%#{order.part_name}%", order.id).order('created_at DESC') if obj.blank? && order.part_name.present?
copied = false
obj.limit(4).each do |o|
SimpleOrderx.production_step_class.where('resource_id = ? AND resource_string = ? AND definition_category = ?', o.id, params[:controller].sub('/', '_'), 'production_step').each do |r|
new_step = SimpleOrderx.production_step_class.new()
new_step = r.dup
new_step.resource_id = order.id
begin
new_step.save
copied = true
rescue => e
flash[:notice] = 'Error in copying production steps: ' + e.message
end
end
return if copied
end if obj.present?
end
In my task file post.rake, I want to reuse a function
def save_post(title, href, source)
post = Post.new(title: title, url: href, source: source)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
end
However, when I define it in this file and re-use it, it returns
NoMethodError: undefined method `save_post' for main:Object
The post.rake looks like this:
task :fetch_post => :environment do
require 'nokogiri'
require 'open-uri'
url = 'http://example.com'
doc = Nokogiri::HTML(open(url) )
puts doc.css("title").text
doc.css(".a").each do |item_info|
title = item_info.text
href = item_info['href']
save_post(title, href)
end
def save_post(title, href)
post = Post.new(title: title, url: href)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
end
end
The content-scraping part works. I just move the post-saving code out, wanting to abstract the method out.
Where should I put the def method?
If you define methods in a rake task, they become accessible globally, which may have undesired side-effects. A cleaner approach is to use an inline lambda (or move the method to some class in the app)
task :fetch_post => :environment do
require 'nokogiri'
require 'open-uri'
save_post = ->(title, href) {
post = Post.new(title: title, url: href)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
}
url = 'http://example.com'
doc = Nokogiri::HTML(open(url) )
puts doc.css("title").text
doc.css(".a").each do |item_info|
title = item_info.text
href = item_info['href']
save_post.call(title, href)
end
end
OOOH~~~,function postion is wrong, like this, it works:
task :fetch_post => :environment do
require 'nokogiri'
require 'open-uri'
def save_post(title, href)
post = Post.new(title: title, url: href)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
end
url = 'http://example.com'
doc = Nokogiri::HTML(open(url) )
puts doc.css("title").text
doc.css(".a").each do |item_info|
title = item_info.text
href = item_info['href']
save_post(title, href)
end
end
You should define method before and outside of task:
task :fetch_post => :environment do
require 'nokogiri'
require 'open-uri'
url = 'http://example.com'
doc = Nokogiri::HTML(open(url) )
puts doc.css("title").text
doc.css(".a").each do |item_info|
title = item_info.text
href = item_info['href']
save_post(title, href)
end
end
def save_post(title, href)
post = Post.new(title: title, url: href)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
end
But i think this logic should be in model.
#app/models/post.rb
class Post < ActiveRecord::Base
def self.save_post(title, href)
post = Post.new(title: title, url: href)
if post.save
puts title + 'saved'
else
puts title + 'not saved'
end
end
end
I am using resque to process a file in the background. It's a CSV file however I get the following error: uninitialized constant ImportFileHelper::CSV
I have tried to require 'csv' and also include CSV neither will work.
require 'csv'
module ImportFileHelper
HOST = ""
USER_NAME = ""
PASSWORD = ""
def self.process_file(file_data, file_name)
init
#file_name = file_name
begin
csv = CSV.parse(file_data, :headers => true)
csv.each do |row|
#first_name = row["FirstName"]
#last_name = row["LastName"]
#email = row["Email"]
#password = "ch#ngeM3!"
#user_group_name = row["GroupName"].split(",")
#store_name = row["StoreName"]
#external_id = row["ExternalID"]
add_user unless #first_name.nil? || #last_name.nil? || #email.nil? || #password.nil? || #first_name.empty? || #last_name.empty? || #email.empty?
end
rescue NoMethodError => no_method_error
log_error_to_db no_method_error
rescue IOError => error
log_error_to_db error
#errors << error.to_s
rescue Exception => ex
log_error_to_db ex
end
prep_soap_responses_for_output
end
def self.init
HTTPI.log = false
#body = { username: USER_NAME, password: PASSWORD }
#errors = []
#existing_users = []
configure_savon
get_all_groups
get_all_stores
end
def self.prep_soap_responses_for_output
[#existing_users, #errors]
end
def self.log_error_to_db(error)
error.backtrace ||= "Not Available"
if error.message.length > 250
error_message = "There was an error"
else
error_message = error.message
end
ErrorLog.create(message: error_message, trace: error.backtrace, file_name: #file_name)
end
def self.get_store_id
#store_id = #stores[#store_name.to_sym]
end
def self.get_all_stores
#stores = { }
client = Savon::Client.new(HOST + "Storews.asmx?wsdl")
body_data = { mall_id: 1, is_return_offline_store: :false }
#body.merge! body_data
begin
response = client.request :get_store_list, body: #body
if response
hash_response = response.to_hash
stores = hash_response[:get_store_list_response][:get_store_list_result][:store]
stores.each do |s|
store = { s[:name].to_sym => s[:store_id] }
#stores.merge! store
end
end
rescue Savon::Error => ex
log_error_to_db error
#errors << error.to_s
end
end
def self.create_adbuilder_user_object
AdbuilderUser.new(#first_name, #last_name, #email, #user_id, #store_id, #store_name, #user_group_name, #group_id, #external_id)
end
def self.configure_savon
Savon.configure do |configure|
configure.log = false
end
end
def self.add_user
body_data = { first_name: #first_name, last_name: #last_name, user_password: #password, email: #email, external_id: #external_id }
#body.merge! body_data
begin
client = Savon::Client.new(HOST + "UserWS.asmx?wsdl")
response = client.request :add_user, body: #body
if response
#user_id = response.body[:add_user_response][:add_user_result]
if #user_group_name
get_group_id
end
if #store_name
#store_id = get_store_id
unless #store_id.to_s =~ /^0$/
adbuilder_user = create_adbuilder_user_object
UserMailer.create_password(adbuilder_user).deliver if adbuilder_user
end
end
end
rescue Savon::Error => error
log_error_to_db error
if error.message == "(soap:Client) 3: A user with the same email login already exists. Please choose a different login."
#existing_users << #email
else
#errors << error.to_s
end
rescue Exception => error
log_error_to_db error
#errors << error.message.to_s
end
end
def self.get_group_id
begin
#user_group_name.each do |group_name|
user_group_id = #groups_info[group_name.downcase.to_sym]
add_user_to_group user_group_id if user_group_id
end
rescue Exception => error
log_error_to_db error
#errors << error.message.to_s
end
end
def self.get_all_groups
#groups_info = {}
begin
client = Savon::Client.new(HOST + "Usergroupws.asmx?wsdl")
response = client.request :get_user_group_list, body: #body
if response
group = response.to_hash
groups = group[:get_user_group_list_response][:get_user_group_list_result][:user_group]
groups.each do |g|
new_hash = { g[:name].gsub(/\s/, "_").downcase.to_sym => g[:user_group_id] }
#groups_info.merge! new_hash
end
end
rescue Savon::Error => error
log_error_to_db
#errors << error.to_s
end
end
def self.add_user_to_group(group_id)
body_data = { user_id: #user_id, user_group_id: group_id }
#body.merge! body_data
begin
client = Savon::Client.new(HOST + "Usergroupws.asmx?wsdl")
response = client.request :add_user_to_group, body: #body
rescue Savon::Error => error
log_error_to_db error
#errors << error.to_s
end
end
end
So as a work around for this I am doing the csv parsing in the resque job file. This is now allowing it to run. Not sure if this is the best way to do it though.
class ProcessFile
#queue = :rts_file_parser
def self.perform(file_data, file_name)
csv = CSV.parse(file_data, :headers => true)
csv.each do |row|
row_data = { first_name: row["FirstName"], last_name: row["LastName"], email: row["Email"], password: "ch#ngeM3!", user_group_name: row["GroupName"].split(","), store_name: row["StoreName"], external_id: row["ExternalID"] }
ImportFileHelper.process_file file_name, row_data
end
end
end
Mind if I claim the answer (via my comment)?
It looks like it might be a scope resolution issue.
Try ::CSV instead of CSV.
Try adding the gem to the gemfile.