I would like to pull datas from GA and improve theses querys by chaining filters and dimensions (if it's possible)
Total of each unique event (by month and label) for an user like :
start_date = 1.month.ago.last_month.beginning_of_month.strftime("%Y-%m-%d")
end_date = 1.month.ago.last_month.end_of_month.strftime("%Y-%m-%d")
metrics = 'ga:uniqueEvents'
dimensions = ['ga:eventAction','ga:eventLabel']
vue = "ga:eventAction==vue;ga:eventLabel==#{user.id}"
telephone = "ga:eventAction==Téléphone;ga:eventLabel==#{user.id}"
mobile = "ga:eventAction==Mobile;ga:eventLabel==#{user.id}"
vues = client.get_ga_data(PROFILE, start_date, end_date, metrics, { dimensions: dimensions, filters: vue })
telephone = client.get_ga_data(PROFILE, start_date, end_date, metrics, { dimensions: dimensions, filters: telephone })
mobile = client.get_ga_data(PROFILE, start_date, end_date, metrics, { dimensions: dimensions, filters: mobile })
Stat.find_or_create_by!(month: start_date, user_id: user.id) do |stat|
stat.vues = vues.totals_for_all_results["ga:uniqueEvents"].to_i,
stat.phone = telephone.totals_for_all_results["ga:uniqueEvents"].to_i,
stat.mobil_phone = mobile.totals_for_all_results["ga:uniqueEvents"].to_i
end
Related
Basically I'd like to return all people whose current job title is X and whose previous job title is Y. As an example, I have a talent whose current emnployment is "Airbnb (company_id = 1)" and whose previous employment is at "Youtube (company_id = 2)".
If I run a query to find talent where current employment is Airbnb:
Talent.joins(:job_histories).where(["job_histories.company_id = ? and job_histories.end_year = ?", 1, "Present"])
I get the person.
If I run a query where previous employment is Youtube (hence the end_year != "Present" below)
Talent.joins(:job_histories).where(["job_histories.company_id = ? and job_histories.end_year != ?", 2, "Present"])
I also get the same person.
However, if I chain them together to find talents where current employer is Airbnb AND previous employer is Youtube, like this:
#talents = Talent.all
#talents = #talents.joins(:job_histories).where(["job_histories.company_id = ? and job_histories.end_year = ?", 1, "Present"])
#talents = #talents.joins(:job_histories).where(["job_histories.company_id = ? and job_histories.end_year != ?", 2, "Present"])
I do not get any results. I've tried several variations of the query but none return anything.
The only way I can get it to work is by using the first query and then looping over each talent to find where job_histories.company_id == 2.
if params[:advanced_current_company] && params[:advanced_previous_company]
#talents = #talents.joins(:job_histories).where(job_histories: { company_id: params[:advanced_current_company] }).distinct if params[:advanced_current_company]
#talents.each do |talent|
talent.job_histories.each do |job_history|
if job_history.company_id == params[:advanced_previous_company][0].to_i
new_talents.append(talent.id)
end
end
end
#talents = Talent.where(id: new_talents)
end
Any direction would be amazing. Thanks!
You had the right idea with a double join of the job_histories, but you need to alias the job_histories table names to be able to differentiate between them in the query, as otherwise activerecord will think it's only one join that needs to be done.
Talent.joins("INNER JOIN job_histories as jh1 ON jh1.talent_id = talents.id")
.joins("INNER JOIN job_histories as jh2 ON jh2.talent_id = talents.id")
.where("jh1.company_id = ? and jh1.end_year = ?", 1, "Present")
.where("jh2.company_id = ? and jh2.end_year != ?", 2, "Present")
I'm using the code from here to scrape the tweets of a few users and export is as a .csv: https://towardsdatascience.com/tweepy-for-beginners-24baf21f2c25
I want to ideally get all the tweets of each user, but it seems to be limited to only the most recent 3200 tweets. Here's the exact code I'm using with trump as an example:
ids = ['realDonaldTrump']
def extract_hashtags(hashtag_list):
final_hashtag = ''
for hashtag in hashtag_list:
final_hashtag = final_hashtag + ' ' + hashtag['text']
return final_hashtag.strip()
#from https://towardsdatascience.com/tweepy-for-beginners-24baf21f2c25
class TweetMiner(object):
result_limit = 20
data = []
api = False
twitter_keys = { #redacted }
def __init__(self, keys_dict=twitter_keys, api=api, result_limit = 20):
self.twitter_keys = keys_dict
auth = tw.OAuthHandler(keys_dict['consumer_key'], keys_dict['consumer_secret'])
auth.set_access_token(keys_dict['access_token_key'], keys_dict['access_token_secret'])
self.api = tw.API(auth)
self.twitter_keys = keys_dict
self.result_limit = result_limit
def mine_user_tweets(self, user,
mine_rewteets=False,
max_pages=5):
data = []
last_tweet_id = False
page = 1
while page <= max_pages:
if last_tweet_id:
statuses = self.api.user_timeline(screen_name=user,
count=self.result_limit,
max_id=last_tweet_id - 1,
tweet_mode = 'extended',
include_retweets=True
)
else:
statuses = self.api.user_timeline(screen_name=user,
count=self.result_limit,
tweet_mode = 'extended',
include_retweets=True)
for item in statuses:
mined = {
'tweet_id': item.id,
'name': item.user.name,
'screen_name': item.user.screen_name,
'retweet_count': item.retweet_count,
'text': item.full_text,
'mined_at': datetime.datetime.now(),
'created_at': item.created_at,
#'time_zone': item._json['time_zone'],
'favourite_count': item.favorite_count,
'hashtags': extract_hashtags(item.entities['hashtags']),
#'links': extract_
'status_count': item.user.statuses_count,
'location': item.place,
'source_device': item.source
}
try:
mined['retweet_text'] = item.retweeted_status.full_text
except:
mined['retweet_text'] = 'None'
try:
mined['quote_text'] = item.quoted_status.full_text
mined['quote_screen_name'] = status.quoted_status.user.screen_name
except:
mined['quote_text'] = 'None'
mined['quote_screen_name'] = 'None'
last_tweet_id = item.id
data.append(mined)
page += 1
return data
#result_limit * max_pages is the no of tweets for each id
miner=TweetMiner(result_limit = 460) #200
counter = 0
counter2 = 0
for id in ids:
try:
print("Fetching tweets of " + id+ " now...")
mined_tweets = miner.mine_user_tweets(user= id, max_pages=460) #100
mined_tweets_df= pd.DataFrame(mined_tweets)
counter2 = counter2 +1
#after 40 tries, pause for 15 mins
if counter2%40==0: #5
print("Couldn't fetch, sleeping for 15 mins")
time.sleep(900) #15 minute sleep time
except:
print(id, 'is invalid or locked')
if counter>0:
final_df = pd.concat([final_df, mined_tweets_df], ignore_index = True)
print("Fetched and added!")
else:
final_df = mined_tweets_df
print("Fetched and added!")
counter +=1
print(final_df)
final_df.to_csv('tweets.csv', encoding='UTF-8')
The number of tweets returned should be 460*460 = 211,600 tweets for each user in ids, but it only returns a total of 3200 tweets per id. Is this limit a strict one built into the API, and if so, is there any way to get more than 3200 tweets per user?
This is a limit built into the Twitter API. The user timeline can only return a maximum of 3200 Tweets (in 200 Tweets per “page”). To retrieve more, you would need to use the premium or enterprise full archive search API.
I want to perform multiple calculations using one query to price_histories table, and finally render some statistics using those prices like average, minimum and maximum etc.
price_histories_controller.rb
price_stats = PriceHistory.where('created_at >= ? AND cast(item_id as integer) = ?', 1.day.ago, params['item_id'])
avg_array = price_stats.group(:day).average(:price).to_a
min_array = price_stats.group(:day).min(:price).to_a
max_array = price_stats.group(:day).max(:price).to_a
count_array = price_stats.group(:day).count(:price).to_a
This is the relevant code that causes the error, i'd like to perform some calculations on a set of grouped data but after the first calculation is done, I keep getting
TypeError (no implicit conversion of Symbol into Integer)
Ideally I would end up with an object like this one to be rendered:
#all_stats = {
average: avg_array,
min: min_array,
max: max_array,
count: count_array
}
render json: #all_stats
This sums up my intentions pretty well, I'm new to ruby and I'd like a solution or a better approach which I'm sure there are.
The following code works fine and I'd like anyone to point me in the right direction to finding out why this works fine and when adding and extra calculation it doesn't:
price_stats = PriceHistory.where('created_at >= ? AND cast(item_id as integer) = ?', 1.day.ago, params['item_id'])
avg_array = price_stats.group(:day).average(:price).to_a
and leads to:
{
"average": [
[
null,
"11666.666666666667"
],
[
"24/4/2019",
"11666.666666666667"
],
[
"24",
"11666.6666666666666667"
],
[
"2051",
"11666.6666666666666667"
]
],
"min": [],
"max": [],
"count": []
}
Other approach:
PriceHistory.select(
"AVG(price) AS average_score,
MIN(price) AS average_min,
MAX(price) AS average_max,
COUNT(*) AS price_count"
).where(
'created_at >= ? AND cast(item_id as integer) = ?',
1.day.ago, params['item_id']
).group(:day)
Error:
ArgumentError (Call `select' with at least one field):
I think this should work:
PriceHistory.where(
'created_at >= ? AND cast(item_id as integer) = ?',
1.day.ago,
params['item_id']
).group(:day).select(
"SUM(price) AS sum_price",
"MAX(price) AS max_price",
"MIN(price) AS min_price",
"AVG(price) AS avg_price",
"day"
)
This will return you an array of records, each which has methods day, sum_price, max_price, min_price, and avg_price.
Note that the names of the SQL functions might be different based on your db
I need a RoR mongoDB query to list articles within a given radius, sorted by created_at.
The challenge is that addresses are saved in separate table and referenced by key/id out of articles. Don't know how to make query with geoNear for this scenario.
Also pagination needed and performant query desirable.
Currently approaching like:
Get addresses within defined radius
Get articles associated to address results out of 1.
sort_by address (geoNear default)
Pagination is making usage of last_address_id. Also here have an issue, as last page is in loop.
#seaches_controller.rb
def index
#addresses =
Address.get_addresses_with_radius(article_search_params).to_a
#address_hash = #addresses.group_by{|a| a['_id'].to_s}
#articles = Article.includes(:gift, :category)
.where( transaction_status:
{
'$nin' => ["concluded"]
},
address_id:
{
:$in => #addresses.map{|a| a['_id'].to_s}
}
).to_a
.sort_by{|m|
#addresses.map{|a|
a['_id']}.index(m['address_id']) }
end
#address.rb
def self.get_addresses_with_radius(params, additional_query={})
#raw query for aggreegate with geoNear
last_maximum_distance = params[:last_maximum_distance] || 0 # in
meeters
radius = params[:radius] || 5000000 #In Meters
query_params = additional_query
if params[:last_address_id]
query_params[:_id] ||= {}
query_params[:_id] = query_params[:_id].merge({ '$ne' =>
(BSON::ObjectId(params[:last_address_id])) } )
end
addresses_in_radius =
Address.collection.aggregate([
{
'$geoNear':
{
near:
{
type: "Point",
coordinates: [ params[:lat].to_f, params[:lon].to_f ]
},
distanceField: "distance_from", #GeoNear Will atomatically distance as distance_from_field
minDistance: last_maximum_distance.to_f,
maxDistance: radius,
query: query_params,
#query:{ 'location.0': {'$ne' =>
params[:last_lat].to_f},'location.1': {'$ne' => params[:last_lon].to_f}},
spherical: true
}
},
{"$limit": params[:per_page].to_i}
])
addresses_in_radius
end
Currently I'm getting the list of articles sorted by addresses/distance, as per default geoNear behavior => should be by created_at.
Pagination is somehow based on addresses => should ideally be based on articles.
Pagination is buggy, as last page is loading in loop => loop-bug to go away.
Not sure if best is to first search for articles and then addresses, or first addresses and then get the articles; relevant note: all within defined radius.
I am currently working on a solution in Grails and I have installed the following security plugins:
Spring Security Core
Spring Security UI
I will basically have a solution with the following security structure:
Super Users
Admins(For different business areas)
Users (within the different business areas)
So basically I installed the Spring Security UI in order to allow the various Business Area Admins manage their own areas, they should be able to use the UI in order to allow them to search only for users in thier own area, create users in their own area and edit users only in their own area. However the spring security UI gives people who have access blanket access do anything.
I have added an extra field to the spring security domain model which is "Area", so I was thinking when the admin is searching for users they would only see users in the same area as them, when they create a user they can only do so for their own area and they can only edit users in their own area.
Below is some code that the spring security UI uses to search for the users, can I modify this in order to only return the users that are in the same area as the admin who is currently logged in? or is there a better way?
def userSearch = {
boolean useOffset = params.containsKey('offset')
setIfMissing 'max', 10, 100
setIfMissing 'offset', 0
def hql = new StringBuilder('FROM ').append(lookupUserClassName()).append(' u WHERE 1=1 ')
def queryParams = [:]
def userLookup = SpringSecurityUtils.securityConfig.userLookup
String usernameFieldName = userLookup.usernamePropertyName
for (name in [username: usernameFieldName]) {
if (params[name.key]) {
hql.append " AND LOWER(u.${name.value}) LIKE :${name.key}"
queryParams[name.key] = params[name.key].toLowerCase() + '%'
}
}
String enabledPropertyName = userLookup.enabledPropertyName
String accountExpiredPropertyName = userLookup.accountExpiredPropertyName
String accountLockedPropertyName = userLookup.accountLockedPropertyName
String passwordExpiredPropertyName = userLookup.passwordExpiredPropertyName
for (name in [enabled: enabledPropertyName,
accountExpired: accountExpiredPropertyName,
accountLocked: accountLockedPropertyName,
passwordExpired: passwordExpiredPropertyName]) {
Integer value = params.int(name.key)
if (value) {
hql.append " AND u.${name.value}=:${name.key}"
queryParams[name.key] = value == 1
}
}
int totalCount = lookupUserClass().executeQuery("SELECT COUNT(DISTINCT u) $hql", queryParams)[0]
Integer max = params.int('max')
Integer offset = params.int('offset')
String orderBy = ''
if (params.sort) {
orderBy = " ORDER BY u.$params.sort ${params.order ?: 'ASC'}"
}
def results = lookupUserClass().executeQuery(
"SELECT DISTINCT u $hql $orderBy",
queryParams, [max: max, offset: offset])
def model = [results: results, totalCount: totalCount, searched: true]
// add query params to model for paging
for (name in ['username', 'enabled', 'accountExpired', 'accountLocked',
'passwordExpired', 'sort', 'order']) {
model[name] = params[name]
}
render view: 'search', model: model
}
EDIT....
I believe it may have something to do with the code below:
def results = lookupUserClass().executeQuery(
"SELECT DISTINCT u $hql $orderBy",
queryParams, [max: max, offset: offset])
I think I just need to alter this statement so that it looks for the list of users where the currently logged in users "Area" is equal to the same area as the users. Can anyone please help me with this??
EDIT 2.....
I have now looked into this and have been able to obtain the users Area and now alls I need to do is to modify the query to the database to look for the users that have the same Area as the admin searching. I have tried the following with no luck, can someone please help me with this as I know this must be simple just cant seem to get there :-S
def user = springSecurityService.currentUser
def userArea = user.area
def hql = new StringBuilder('FROM ').append(lookupUserClassName()).append(' u WHERE 1=1 AND u.area = userArea')
EDIT 3.......
Thanks so much half of my problem is solved lol, now just the Ajax piece:
I have tried the below code in order to modify the search for the Ajax function to only return results where the Area of the user is the same as the currently logged in user:
String username = params.term
String usernameFieldName = SpringSecurityUtils.securityConfig.userLookup.usernamePropertyName
def user = springSecurityService.currentUser
setIfMissing 'max', 10, 100
def results = lookupUserClass().executeQuery(
"SELECT DISTINCT u.$usernameFieldName " +
"FROM ${lookupUserClassName()} u " +
"WHERE LOWER(u.$usernameFieldName) LIKE :name AND LOWER(u.area) = :area " +
"ORDER BY u.$usernameFieldName",
[name: "${username.toLowerCase()}%"],
[area: "user.area"],
[max: params.max])
Also tried changing the param as below:
[area: user.area]
The controller is building an HQL query, so you can't just say "WHERE u.area = userArea", you'll need to use a named parameter and put the value in the queryParams map
def user = springSecurityService.currentUser
def hql = new StringBuilder('FROM ').append(lookupUserClassName()).append(
' u WHERE u.area = :userArea ')
def queryParams = [userArea:user.area]
For the second part of the problem (the Ajax bit), I doubt you need the LOWER conversion, and also you need to put all your query parameters into one map (the second map parameter is just for the pagination settings):
def results = lookupUserClass().executeQuery(
"SELECT DISTINCT u.$usernameFieldName " +
"FROM ${lookupUserClassName()} u " +
"WHERE LOWER(u.$usernameFieldName) LIKE :name AND u.area = :area " +
"ORDER BY u.$usernameFieldName",
[name: "${username.toLowerCase()}%", area:user.area],
[max: params.max])
If you really do want the area check to be case-insensitive then leave it as LOWER(u.area) = :area but then you also need to convert the value you are testing against to lower case:
[name: "${username.toLowerCase()}%", area:user.area.toLowerCase()],