syntax error, unexpected tLABEL when running a custom rake task - ruby-on-rails

I'm working on something to scrape real estate data from a certain website. It works as a standalone .rb file while saving to a JSON file. But I want this to run on Heroku and save the data to MongoDB.
Problem:
I keep getting the following errors when running:
rake aborted!
SyntaxError: /Users/user/Dropbox/Development/Rails/booyah/lib/tasks/properties_for_sale.rake:35: syntax error, unexpected tLABEL
street_name: #street_name,
^
/Users/user/Dropbox/Development/Rails/booyah/lib/tasks/properties_for_sale.rake:41: syntax error, unexpected tLABEL, expecting '='
bedrooms: #rooms[1],
^
/Users/user/Dropbox/Development/Rails/booyah/lib/tasks/properties_for_sale.rake:42: syntax error, unexpected tLABEL, expecting '='
number_of_floors: #number_of_floors,
^
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/engine.rb:654:in `load'
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/engine.rb:654:in `block in run_tasks_blocks'
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/engine.rb:654:in `each'
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/engine.rb:654:in `run_tasks_blocks'
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/application.rb:362:in `run_tasks_blocks'
/Users/user/.rvm/gems/ruby-2.1.1/gems/railties-4.1.1/lib/rails/engine.rb:449:in `load_tasks'
/Users/user/Dropbox/Development/Rails/booyah/Rakefile:6:in `<top (required)>'
/Users/user/.rvm/gems/ruby-2.1.1/bin/ruby_executable_hooks:15:in `eval'
/Users/user/.rvm/gems/ruby-2.1.1/bin/ruby_executable_hooks:15:in `<main>'
This is the code I'm using:
require 'mechanize'
namespace :properties_for_sale do
desc "Scrape all properties currently for sale"
task :start => :environment do
a = Mechanize.new
#a2 = Mechanize.new
#i = 1
BASE_URL = 'http://www.funda.nl'
def scrape_objects_on_page(page)
objects_on_page = page.search('//*[contains(concat( " ", #class, " " ), concat( " ", "object-street", " " ))]')
objects_on_page.each do |object|
#a2.get(BASE_URL + object[:href] + 'kenmerken/') do |page_2|
break if page_2.title == '404 - Pagina niet gevonden'
#street_name = page_2.search('//*[#id="main"]/div[1]/div/div/div/h1').text.strip
#price = page_2.search('//*[#id="main"]/div[1]/div/div/div/p[2]/span/span').text.strip.gsub("€ ", "").gsub(".", "").to_i
#url = page_2.uri.to_s
#living_area = page_2.search('//*[#id="twwo13"]/td/span[1]').text.strip.gsub(" m²", "").to_i
#content = page_2.search('//*[#id="twih12"]/td/span[1]').text.strip.gsub(" m³", "").to_i
#rooms = page_2.search('//*[#id="aaka12"]/td/span[1]').text.strip.scan(/\d/).to_i
#number_of_floors = page_2.search('//*[#id="twva12"]/td/span[1]').text.strip.to_i
#year = page_2.search('//*[#id="boja12"]/td/span[1]').text.strip.to_i
#broker = page_2.search('//*[contains(concat( " ", #class, " " ), concat( " ", "rel-info", " " ))]//h3').text.strip
#city = page_2.search('//*[#id="nav-path"]/div/p[1]/span[4]/a/span').text.strip
#district = page_2.search('//*[#id="nav-path"]/div/p[1]/span[5]/a/span').text.strip
#province = page_2.search('//*[#id="nav-path"]/div/p[1]/span[3]/a/span').text.strip
#type_of_property = page_2.search('//*[#id="soap12"]/td/span[1] | //*[#id="sowo12"]/td/span[1] | //*[#id="twsp12"]/td/span[1]').text.strip
Property.create = (
street_name: #street_name,
price: #price,
url: #url,
living_area: #living_area,
content: #content,
rooms: #rooms[0],
bedrooms: #rooms[1],
number_of_floors: #number_of_floors,
year: #year,
broker: #broker,
city: #city,
district: #district,
province: #province,
type_of_property: #type_of_property
)
puts Property.last
end
end
end
loop do
a.get("http://www.funda.nl/koop/rotterdam/sorteer-datum-af/p#{#i}/") do |page|
#end = page.search('//h3').text == 'Geen koopwoningen gevonden die voldoen aan uw zoekopdracht' ? true : false
scrape_objects_on_page(page) unless #end == true
#i = #i + 1
end
break if #end
end
puts "==================================================================================="
puts "# Done scraping #{#i - 1} pages and collected #{#all_objects_array.length} objects."
puts "==================================================================================="
end
end
This is what my Property model looks like (MongoMapper):
class Property
include MongoMapper::Document
key :street_name, String
key :price, Integer
key :url, String
key :living_area, Integer
key :content, Integer
key :rooms, Integer
key :bedrooms, Integer
key :number_of_floors, Integer
key :year, Integer
key :broker, String
key :city, String
key :district, String
key :province, String
key :type_of_property, String
end
What am I doing wrong?

You have a typo. Remove the equal sign between Property.create and the parens. Like below:
Property.create(
street_name: #street_name,
price: #price,
url: #url,
living_area: #living_area,
content: #content,
rooms: #rooms[0],
bedrooms: #rooms[1],
number_of_floors: #number_of_floors,
year: #year,
broker: #broker,
city: #city,
district: #district,
province: #province,
type_of_property: #type_of_property
)
Also, it might be better to store the #create call in a variable instead of calling Property.last. That way, you don't have to issue another query.

Related

Why am I getting "no implicit conversion of String into Integer" when trying to get Nested JSON attribute?

My Rails app is reading in JSON from a Bing API, and creating a record for each result. However, when I try to save one of the nested JSON attributes, I'm getting Resource creation error: no implicit conversion of String into Integer.
The JSON looks like this:
{
"Demo": {
"_type": "News",
"readLink": "https://api.cognitive.microsoft.com/api/v7/news/search?q=european+football",
"totalEstimatedMatches": 2750000,
"value": [
{
"provider": [
{
"_type": "Organization",
"name": "Tuko on MSN.com"
}
],
"name": "Hope for football fans as top European club resume training despite coronavirus threat",
"url": "https://www.msn.com/en-xl/news/other/hope-for-football-fans-as-top-european-club-resume-training-despite-coronavirus-threat/ar-BB12eC6Q",
"description": "Bayern have returned to training days after leaving camp following the outbreak of coronavirus. The Bundesliga is among top European competitions suspended."
}
}
The attribute I'm having trouble with is [:provider][:name].
Here's my code:
def handle_bing
#terms = get_terms
#terms.each do |t|
news = get_news(t)
news['value'].each do |n|
create_resource(n)
end
end
end
def get_terms
term = ["European football"]
end
def get_news(term)
accessKey = "foobar"
uri = "https://api.cognitive.microsoft.com"
path = "/bing/v7.0/news/search"
uri = URI(uri + path + "?q=" + URI.escape(term))
request = Net::HTTP::Get.new(uri)
request['Ocp-Apim-Subscription-Key'] = accessKey
response = Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
http.request(request)
end
response.each_header do |key, value|
# header names are coerced to lowercase
if key.start_with?("bingapis-") or key.start_with?("x-msedge-") then
puts key + ": " + value
end
end
return JSON(response.body)
end
def create_resource(news)
Resource.create(
name: news['name'],
url: news['url'],
description: news['description'],
publisher: news['provider']['name']
)
end
I looked at these questions, but they didn't help me:
Extract specific field from JSON nested hashes
No implicit conversion of String into Integer (TypeError)?
Why do I get "no implicit conversion of String into Integer (TypeError)"?
UPDATE:
I also tried updating the code to:
publisher: news['provider'][0]['name'], but I received the same error.
because "provider" is an array.
it should be accessed with index.
[:value][0][:provider][0][:name]
same goes with "value".

How to parse and loop through JSON sub-children

I have a JSON string:
{
"normal_domains":[{
"urls [
"domain1.com",
"domain2.com"
],
"id":3,
"find":"ama",
"type":"text"
}
],
"premium_domains":[{
"urls":[
"domain3.com",
"domain4.com"
],
"id":1,
"find":"amg",
"type":"text"
}
]
}
I want to output a list for each domain in the hash with corresponding attributes:
Domain type: normal_domains
Domain: domain3.com
ID: 3
Find: ama
-- for each domain --
The code I have is this, but I cannot get it working. It returns NoMethodError: undefined method [] for nil:NilClass:
from_api = '{"normal_domains":[{"urls":["domain1.com","domain2.com"],"id":3,"find":"ama","type":"text"}],"premium_domains":[{"urls":["domain3.com","domain4.com"],"id":1,"find":"amg","type":"text"}]}'
result = JSON.parse from_api
result.each do |child|
loop_index = 0
child.each do |sub_child|
puts "Domain type: #{child}"
puts "Domain: #{sub_child[loop_index]['urls']}"
puts "ID: #{sub_child[loop_index]['id']}"
puts "Find: #{sub_child[loop_index]['find']}"
loop_index += 1
end
end
The hash returned from JSON.parse does not have a .each method.
Imagine your input hash in a more organized way:
{
"normal_domains":[ {
"urls [
"domain1.com",
"domain2.com"
],
"id":3,
"find":"ama",
"type":"text"
}],
"premium_domains":[{
"urls":[
"domain3.com",
"domain4.com"
],
"id":1,
"find":"amg",
"type":"text"
}]
}
You code should be:
result = JSON.parse from_api
result.keys.each do |domain_type|
childArray = result[domain_type]
childArray.each do |child|
urls = child["urls"]
urls.each do |url|
puts "Domain type: #{domain_type}"
puts "Domain: #{url}"
puts "ID: #{child['id']}"
puts "Find: #{child['find']}"
end
end
end
If you want to iterate over an array in a style that is common in C i.e. using array indexes, you should do it like
urls = domain['urls']
(0..urls.length).each do |i|
puts " URL: #{urls[i]}"
end
or at least handle the case when indexing an array element returns nil when the code tries to access data beyond what has been entered in the array. Using array indexes is often unnecessary since iterators can be used.
Using iterators, one does not need the indexes and there is no need to check if the access is beyond the boundaries of a container.
result.each do |key,value|
puts "Domain type: #{key}"
value.each do |domain|
id = domain['id']
find = domain['find']
type = domain['type']
puts " ID: #{id}"
puts " Find: #{find}"
puts " Type: #{type}"
domain['urls'].each do |url|
puts " URL: #{url}"
end
end
end

Ruby on Rails Controller Error in Search

I am facing the below error after I removed two fields middle_name, last_name from the Student migration thru another migration. below are the errors.
Searchlogic::NamedScopes::OrConditions::UnknownConditionError in StudentController#advanced_search
The condition 'last_name' is not a valid condition, we could not find any scopes that match this.
RAILS_ROOT: /root/ansipro342
Application Trace | Framework Trace | Full Trace
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:96:in `interpolate_or_conditions'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:75:in `each'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:75:in `interpolate_or_conditions'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:36:in `or_conditions'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:19:in `or_condition?'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/named_scopes/or_conditions.rb:10:in `condition?'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/scopes.rb:10:in `scope?'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/method_missing.rb:16:in `method_missing'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/conditions.rb:19:in `send'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/conditions.rb:19:in `conditions='
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/conditions.rb:15:in `each'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/conditions.rb:15:in `conditions='
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/date_parts.rb:19:in `conditions='
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/base.rb:18:in `initialize'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/implementation.rb:10:in `new'
/root/ansipro342/vendor/plugins/searchlogic/lib/searchlogic/search/implementation.rb:10:in `search'
/root/ansipro342/app/controllers/student_controller.rb:816:in `advanced_search'
Request
Parameters:
{"search"=>{"order"=>"",
"first_name_or_middle_name_or_last_name_or_admission_no_like"=>"Imran"},
"commit"=>"Search"}
Following is the code area from Student controller where line 816 exists
def advanced_search
#search = Student.search(params[:search])
unless params[:search].present?
#batches = Batch.all
else
if params[:search].present?
#students = Array.new
if params[:advv_search].present? and params[:advv_search][:course_id].present?
unless params[:search][:batch_id_equals].present?
params[:search][:batch_id_in] = Batch.find_all_by_course_id(params[:advv_search][:course_id]).collect{|b|b.id}
end
end
if params[:search][:is_active_equals]=="true"
#students = Student.ascend_by_first_name.search(params[:search]).paginate(:page => params[:page],:per_page => 30)
elsif params[:search][:is_active_equals]=="false"
#students = ArchivedStudent.ascend_by_first_name.search(params[:search]).paginate(:page => params[:page],:per_page => 30)
else
#students = [{:student => {:search_options => params[:search], :order => :first_name}},{:archived_student => {:search_options => params[:search], :order => :first_name}}].model_paginate(:page => params[:page],:per_page => 30)#.sort!{|m, n| m.first_name.capitalize <=> n.first_name.capitalize}
end
#searched_for = ''
#searched_for += "<span>#{t('name')}/#{t('admission_no')}: " + params[:search][:first_name_or_admission_no_like].to_s + "</span>" if params[:search][:first_name_or_admission_no_like].present?
#searched_for += "<span>#{t('name')}: " + params[:search][:first_name_like].to_s + "</span>" if params[:search][:first_name_like].present?
#searched_for += " <span>#{t('admission_no')}: " + params[:search][:admission_no_equals].to_s + "</span>" if params[:search][:admission_no_equals].present?
if params[:advv_search].present? and params[:advv_search][:course_id].present?
course = Course.find(params[:advv_search][:course_id])
batch = Batch.find(params[:search][:batch_id_equals]) unless (params[:search][:batch_id_equals]).blank?
#searched_for += "<span>#{t('course_text')}: " + course.full_name + "</span>"
#searched_for += "<span>#{t('batch')}: " + batch.full_name + "</span>" if batch.present?
end
Please help I dont know where the last_name exists that its showing in error, I am very new to ruby on rails nd need this importantly.
Thanks in advance.
Thanks for the replies, The problem solved.
The problem was in the index migration, actually I removed the columns from the create migration but it was already there in index migration.
Dude that code is huge and very unlikely that someone will understand it by reading it one time..
Install rubocop or some other lint tool and clean that mess.

Ruby on Rails: Assigning attribute values to generic model

I am trying to write a ruby on rails function that will create a new object for any model. Here is what I have so far
def create_populated_object(model)
test_object = model.new
model.columns.each do |column|
attr_name = column.name
attr_type = column.type
#test_object.assign_attributes(attr_name+':'+ "9")
puts "#{':'+attr_name} => #{attr_type}"
if attr_type.to_s == 'integer'
b = '{:' + attr_name.to_s + '=>' + 9.to_s + '}'
puts b
test_object.assign_attributes(b)
puts "it worked"
elsif attr_type.to_s == 'string'
puts "string"
elsif attr_type.to_s == 'datetime'
puts "date time"
elsif attr_type.to_s == 'boolean'
puts "boolean"
elsif attr_type.to_s == 'text'
puts "text"
else
puts "UNKNOWN ATTRIBUTE TYPE"
end
end
puts test_object
end
In my example, id is the first attribute of the model. I try to assign the value 9 to it, but I keep getting this error:
NoMethodError: undefined method `stringify_keys' for "{:id=>9}":String
Anyone know how to fix this?
You need to send a Hash object instead of a string to the method:
b = { attr_name => 9 }
test_object.assign_attributes(b)
assign_attributes expects a hash of attributes to be passed to it. You are passing it a string. Would it be problematic to simply say b = {attr_name.to_sym => 9}?

Arel producing different SQL output

I'm writing a gem for Rails that makes use of Arel. I have run into a case where the output generated by a subnode is different depending on how the output is generated. Below is a test case. Is this a bug or am I doing something wrong?
it 'should generate the same output' do
def ts_language; 'english'; end
def searchable_columns; [:id, :name]; end
def ts_column_sets
column_sets = if searchable_columns[0].is_a?(Array)
searchable_columns.map do |array|
array.map { |c| Table.new(:users)[c] }
end
else
searchable_columns.map { |c| Table.new(:users)[c] }.map { |x| [x] }
end
end
def ts_vectors
ts_column_sets.map do |columns|
coalesce = columns[1..-1].inject(columns[0]) do |memo, column|
Arel::Nodes::InfixOperation.new('||', memo, column)
end
coalesce = Arel::Nodes::InfixOperation.new('::', Arel::Nodes::NamedFunction.new('COALESCE', [coalesce, '']), Arel::Nodes::SqlLiteral.new('text'))
Arel::Nodes::NamedFunction.new('to_tsvector', [ts_language, coalesce])
end
end
def ts_query(query)
querytext = query.is_a?(Array) ? query.map(&:to_s).map(&:strip) : query.to_s.strip.split(" ")
querytext = querytext[1..-1].inject(querytext[0]) { |memo, c| memo + ' & ' + c }
querytext << ':*'
querytext = Arel::Nodes::InfixOperation.new('::', querytext, Arel::Nodes::SqlLiteral.new('text'))
Arel::Nodes::NamedFunction.new('to_tsquery', ['english', querytext])
end
node = Arel::Nodes::InfixOperation.new('##', ts_vectors[0], ts_query(0))
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text)', node.left.to_sql
assert_equal 'to_tsquery(\'english\', \'0:*\' :: text)', node.right.to_sql
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, \'0:*\' :: text)', node.to_sql
end
The line
assert_equal 'to_tsquery(\'english\', \'0:*\' :: text)', node.right.to_sql
is correct but the line
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, \'0:*\' :: text)', node.to_sql
results in an error and the output is:
'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, 0 :: text)'

Resources