This code was working fine few moments ago but now its not working?
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import time
import random
consumer_key = ""
consumer_secret = ""
access_token_key = ""
access_token_secret = ""
Coords = dict()
Place = dict()
PlaceCoords = dict()
XY = []
class StdOutListener(StreamListener):
""" A listener handles tweets that are the received from the stream.
This is a basic listener that inserts tweets into MySQLdb.
"""
def on_status(self, status):
#print "Tweet Text: ",status.text
text = status.text
#print "Time Stamp: ",status.created_at
try:
Coords.update(status.coordinates)
XY = (Coords.get('coordinates')) #Place the coordinates values into a list 'XY'
#print "X: ", XY[0]
#print "Y: ", XY[1]
except:
#Often times users opt into 'place' which is neighborhood size polygon
#Calculate center of polygon
Place.update(status.place)
PlaceCoords.update(Place['bounding_box'])
Box = PlaceCoords['coordinates'][0]
XY = [(Box[0][0] + Box[2][0])/2, (Box[0][1] + Box[2][1])/2]
#print "X: ", XY[0]
#print "Y: ", XY[1]
pass
# Comment out next 4 lines to avoid MySQLdb to simply read stream at console
#print {"status_id":status.id_str,"timestamp":status.created_at,"location X":XY[0],"location Y":XY[1],"text":text}
print status.id_str,status.created_at,XY[0],XY[1],text
def main():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token_key, access_token_secret)
stream = Stream(auth, l, timeout=30)
#sleep
nsecs = 2
#Only records 'locations' OR 'tracks', NOT 'tracks (keywords) with locations'
while True:
try:
# Call tweepy's userstream method
#stream.filter(locations=[-122.75,36.8,-121.75,37.8,-74,40,-73,41],languages=['es','tr','ko','fr','ru','de','ja','it','pt'], async=False)##These coordinates are approximate bounding box around USA
stream.filter(locations=[-180,-90,180,90],async=False)
#stream.filter()
#stream.filter(track=['obama'])## This will feed the stream all mentions of 'keyword'
break
except Exception, e:
print Exception , e
# Abnormal exit: Reconnect
#nsecs=random.randint(30)
print "Reconnecting ",nsecs
time.sleep(nsecs)
if __name__ == '__main__':
main()
Is there any other way to collect streaming data based on locations?
No, using the locations parameter is the only way. However, I would strongly advise against your catch-all except statement in the StdOutListener. It is most likely catching a different type of error than the one you expect, masking a problem.
Related
app.py
from flask import Flask, jsonify, request,render_template
import pickle
app = Flask(__name__,template_folder='template')
# load model
model = pickle.load(open("model.pkl",'rb'))
# app
#app.route('/')
def home():
return render_template('recommendation.html')
# routes
#app.route('/api', methods=['POST'])
def predict():
result=request.form
query_user_name=result["user name"]
user_input = {'query':query_user_name}
output_data=model(query_user_name)
print(output_data)
# send back to browser
output ={output_data}
return f'<html><body><h1>{output_data}</h1><form action="/"><button type="submit">back </button> </form></body></html>'
if __name__ == '__main__':
app.run(debug=True)
I am deploying it using a model made from following function,not a pre-existing model having predict attribute.
def model(user):
recommended_list=[]
top_list=[]
x = data.iloc[data.loc[data.Users == user].index[0],2:]
similar = np.array([(data.iloc[i,0],weight_factor(x,data.iloc[i, 2:])) for i in range(0,data.shape[0],1)])
index= np.argsort( similar[:,1] )
index=index[::-1]
similar=similar[index]
neighbours = similar[similar[:,1].astype(float) > 0.6] #Taking threshold as 0.6
for i in range(0,len(neighbours),1):
for j in range(2,len(data.columns),1):
if data.iloc[data.loc[data.Users == neighbours[i][0]].index[0],j]==1 and data.iloc[data.loc[data.Users == user].index[0],j]==0:
recommended_list.append(data.columns[j])
if (len(neighbours)>10):
for i in range(0,10,1): #Top 10 neighbours
top_list.append(neighbours[i][0])
else:
for i in range(len(neighbours)):
top_list.append(neighbours[i][0])
if user in top_list: #Remove the user of which we are asked to find neighbours,each user is always strongly correlated with itself and its of no use to us.
top_list.remove(user)
print(" ")
print("Top users similar to this user are:")
print(" ")
for i in range(0,len(top_list),1):
print(top_list[i])
print(" ")
print("Users similar to this user liked these products too:")
print(" ")
recommended_array=np.unique(np.array(recommended_list))
for i in range(0,len(recommended_array),1):
print(recommended_array[i])
How to deploy it using flask,my output is not being shown in the window,though on deploying the home page is being shown and input is being taken.
your function "model(user)" must return somethings
example:
def model(user):
# code
return somethings
I am following the guide here
Currently this is the model:
SOS_token = 0
EOS_token = 1
class Lang:
def __init__(self, name):
self.name = name
self.word2index = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2index:
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1
def unicodeToAscii(s):
return ''.join(
c for c in unicodedata.normalize('NFD', s)
if unicodedata.category(c) != 'Mn'
)
# Lowercase, trim, and remove non-letter characters
def normalizeString(s):
s = unicodeToAscii(s.lower().strip())
s = re.sub(r"([.!?])", r" \1", s)
s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
return s
def readLangs(lang1, lang2, reverse=False):
print("Reading lines...")
# Read the file and split into lines
lines = open('Scribe/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\
read().strip().split('\n')
# Split every line into pairs and normalize
pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
# Reverse pairs, make Lang instances
if reverse:
pairs = [list(reversed(p)) for p in pairs]
input_lang = Lang(lang2)
output_lang = Lang(lang1)
else:
input_lang = Lang(lang1)
output_lang = Lang(lang2)
return input_lang, output_lang, pair
MAX_LENGTH = 5000
eng_prefixes = (
"i am ", "i m ",
"he is", "he s ",
"she is", "she s ",
"you are", "you re ",
"we are", "we re ",
"they are", "they re "
)
def filterPair(p):
return len(p[0].split(' ')) < MAX_LENGTH and \
len(p[1].split(' ')) < MAX_LENGTH and \
p[1].startswith(eng_prefixes)
def filterPairs(pairs):
return [pair for pair in pairs if filterPair(pair)]
def prepareData(lang1, lang2, reverse=False):
input_lang, output_lang, pairs = readLangs(lang1, lang2, reverse)
print("Read %s sentence pairs" % len(pairs))
pairs = filterPairs(pairs)
print("Trimmed to %s sentence pairs" % len(pairs))
print("Counting words...")
for pair in pairs:
input_lang.addSentence(pair[0])
output_lang.addSentence(pair[1])
print("Counted words:")
print(input_lang.name, input_lang.n_words)
print(output_lang.name, output_lang.n_words)
return input_lang, output_lang, pairs
The difference between what I'm trying to do and the guide is that I'm trying to insert my input languages as list of strings instead of reading them from a file:
pairs=['string one goes like this', 'string two goes like this']
input_lang = Lang(pairs[0][0])
output_lang = Lang(pairs[1][1])
But I it seems like when I try to count the number of words input_lang.n_words in my string I always get 2.
Is there something I'm missing in calling the class Lang?
Update:
I ran
language = Lang('english')
for sentence in pairs: language.addSentence(sentence)
print (language.n_words)
and that gave me the number of words in pairs
Though, that doesn't give me input_lang and output_lang like the guide did:
for pair in pairs:
input_lang.addSentence(pair[0])
output_lang.addSentence(pair[1])
So first of all you are initialising the Lang object with calls to pairs[0][0] and pairs[1][1] which is the same as Lang('s') and Lang('t')
The Lang object is supposed to be an object that stores information about a language so I would expect you need to only initialise it once with Lang('english') and then add the sentences from you dataset to the Lang object with the Lang.addSentence function.
Right now you aren't loading your dataset into the Lang object at all so when you want to know language.n_words it is just the initial value it gets when the object is created self.n_words = 2 # Count SOS and EOS
None of what you are doing in your question makes any sense, but I think what you want is the following:
language = Lang('english')
for sentence in pairs: language.addSentence(sentence)
print (language.n_words)
This is my first attempt to extract tweets using twitter api and tweepy. When I execute my code it keep printing 401 every time in a new line. What am I doing wrong is I am not able to figure out. Any help is appreciated.
import tweepy
import json
access_token = ""
access_token_secret = ""
consumer_key = ""
consumer_secret = ""
auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
class MyStreamListener(tweepy.StreamListener):
def __init__(self, api=None):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
self.file = open("tweets.txt", "w")
def on_status(self, status):
tweet = status._json
self.file.write( json.dumps(tweet) + '\n' )
self.num_tweets += 1
if self.num_tweets < 100:
return True
else:
return False
self.file.close()
def on_error(self, status):
print(status)
l = MyStreamListener()
stream=tweepy.Stream(auth,l)
stream.filter()
tweets_data_path = 'tweets.txt'
tweets_file = open(tweets_data_path, "r")
tweets_data = []
for line in tweets_file:
tweet = json.loads(line)
tweets_data.append(tweet)
tweets_file.close()
print(tweets_data[0].keys())
Go to your twitter account settings and change timezone to that as of your computer. Then, go to twitter app settings and generate new consumer key and new access token. These newly generated keys and tokens you should use to avoid 401 error.
I'm trying to get back into Python and I'm once again stuck with this problem I've had before of making objects accessible to one another. In this simple example I am displaying a panel with a button and a text box. Clicking on the text box calls a function which queries a database and returns a cursor with the retrieved data. I need to make it so that either the LookupSQL function or the ShowClientData function can write this output, in a loop, to the Text box. The TextBox (outputBox) is unknown to any other functions currently. How do I make it so that the other functions know what it is?
import wx
import pypyodbc
conn = pypyodbc.connect(driver='{SQL Server}', server='.', database='TheDB', uid='sa', pwd='Pass')
class Audit(wx.Frame):
def __init__(self, *args, **kwargs):
super(Example, self).__init__(*args, **kwargs)
self.InitUI()
def InitUI(self):
panel = wx.Panel(self)
hbox = wx.BoxSizer()
sizer = wx.GridSizer(6,1,2,2)
btn1 = wx.Button(panel, label='Clients')
outputBox = wx.TextCtrl(panel, -1, style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)
sizer.AddMany([btn1, btn2, btn3, btn4, btn5, btn6])
hbox.Add(sizer, 0, wx.ALL, 15)
hbox.Add(outputBox, 1, wx.EXPAND)
panel.SetSizer(hbox)
btn1.Bind(wx.EVT_BUTTON, self.ShowClientData)
self.SetSize((800, 600))
self.SetTitle('Audit View')
self.Centre()
self.Show(True)
def ShowClientData(self, event):
SQL = 'select * from V_UpdatedClient'
recursor = lookupSQL(SQL)
for row in recursor:
rChange = row[0]
rItemType = row[1]
rPK = row[2]
rItemCode = row[3]
rFieldName = row[4]
rOldValue = row[5]
rNewValue = row[6]
rUpdateDate = row[7]
rUserName = row[8]
print('%s %s %s %s %s %s %s %s %s' % (rChange, rItemType, rPK, rItemCode, rFieldName, rOldValue, rNewValue, rUpdateDate, rUserName))
def lookupSQL(SQLString):
cursor = conn.cursor()
cursor.execute(SQLString)
return cursor
cursor.close()
def main():
ex = wx.App()
Audit(None)
ex.MainLoop()
if __name__ == '__main__':
main()
What you are looking for is called data attributes.
self.outputBox = wx.TextCtrl(panel, -1, style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)
And then within ShowClientData you can write
self.outputBox.AppendText("some text")
As long as you have that self reference, you can access its attributes.
Edit:
When you do the above change, you can't refer to the text box by just outputBox anymore, you should instead access it via self:
hbox.Add(self.outputBox, 1, wx.EXPAND)
Declaring it as globally is very bad!
Soooo I have been working on a script I took from ArcGIS Blueprints:
http://file.allitebooks.com/20151230/ArcGIS%20Blueprints.pdf
It should convert geolocated tweets into a geodatabase. I have the Twitter Streaming API already operational, and been playing with different ways to extract x/y, but keep coming back to this script, every so often, hoping I can get it running with no luck. I am stuck on a "List Index Out of Range" error. If anyone is gracious enough to offer some ideas on how I can get by this error I will be forever grateful. If nothing else this endeavor has exploited my shortcomings with Python and Arcpy, and hopefully it will round me out in the long run. For right now, I sure would like to get some mileage out of this script and the work Ive invested into it. Thank you!
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import arcpy
import sys
import time
consumer_key = 'xxx'
consumer_secret = 'xxx'
token_key = 'xxx'
token_secret = 'xxx'
class StdOutListener(StreamListener):
def __init__(self, start_time, featureClass, time_limit):
super(StdOutListener, self).__init__()
self.time = start_time
self.limit = time_limit
self.featureClass = featureClass
def on_status(self, status):
while (time.time() - self.time) < self.limit:
if status.geo is not None:
dictCoords = status.geo
listCoords = dictCoords['coordinates']
latitude = listCoords[0]
longitude = listCoords[1]
cursor =arcpy.da.InsertCursor(self.featureClass,"SHAPE#XY"))
cursor.insertRow([(longitude,latitude)])
print(str(listCoords[0]) + "," + str(listCoords[1]))
return True
else:
print "No coordinates found"
return True
start_time = time.time()
arcpy.env.workspace = "c:\ArcGIS_Blueprint_Python\data\Twitter\TweetInformation.gdb" "
def main():
try: #new
featureClass = sys.argv[1]
monitorTime = sys.argv[2]
monitorTime = monitorTime * 3600
sr = arcpy.SpatialReference(4326)
arcpy.env.overwriteOutput = True
arcpy.CreateFeatureclass_management(arcpy.env.workspace,
featureClass, "POINT", spatial_reference=sr)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(token_key, token_secret)
stream = Stream(auth, StdOutListener(start_time, featureClass,
time_limit=monitorTime)) #172800
stream.filter(track=['car'])
except Exception as e:
print(e.message)
if __name__ == '__main__':
main()