I am trying to acces the Streaming API, filter it by some terms and then print out the results, using Tweepy. However I am getting the following error:
File "/usr/local/lib/python2.6/dist-packages/tweepy-1.7.1-py2.6.egg/tweepy/streaming.py", line 110, in _run
resp = conn.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 349, in _read_status
line = self.fp.readline()
File "/usr/lib/python2.6/socket.py", line 397, in readline
data = recv(1)
socket.error: [Errno 104] Connection reset by peer
With the following code...
import sys
import tweepy
from textwrap import TextWrapper
from tweepy.streaming import StreamListener, Stream
consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''
auth1 = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth1.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth1)
class StreamListener(tweepy.StreamListener):
status_wrapper = TextWrapper(width=60, initial_indent=' ', subsequent_indent=' ')
def on_status(self, status):
try:
print self.status_wrapper.fill(status.text)
print '\n %s %s via %s\n' % (status.author.screen_name, status.created_at, status.source)
except Exception, e:
pass
def main():
l = StreamListener()
streamer = tweepy.Stream(auth=auth1, listener=l, timeout=3000000000 )
setTerms = ['hello', 'goodbye', 'goodnight', 'good morning']
streamer.filter(None,setTerms)
if __name__ == "__main__":
main()
Does anyone know how to solve it?
Thanks...
The reason was SSL, it seems to be forced by twitter now...
Related
I am building a state machine node(ROS2 Action_client) that interact with Planner node(ROS2 Action_server). In the execution callback of the planner node I need to publish a topic (from the same node). Is it possible to publish a topic while action is running?
class PlanningProblemServer(Node):
def __init__(self): super().__init__('planning_problem_server')
self._action_server = ActionServer(self, PlanAid, 'planning_problem_sm',
execute_callback=self.execute_callback,
goal_callback=self.goal_callback,
cancel_callback=self.cancel_callback)
self._publisher = self.create_publisher(Bool, 'Planning_problem_data')
def goal_callback(self, goal_request): #Some code
def cancel_callback(self, goal_handle): #Some code
def execute_callback(self, goal_handle):
feedback_msg.feedback = "loading planning problem...!"
goal_handle.publish_feedback(feedback_msg)
msg = String()
msg.data = "abc"
self._publisher.publish(msg)
goal_handle.set_succeeded()
return result
def main(args=None):
rclpy.init(args=args)
minimal_action_server = PlanningProblemServer()
rclpy.spin(minimal_action_server)
minimal_action_server.destroy()
rclpy.shutdown()
if __name__ == '__main__': main()
with above code I am getting following error,
Traceback (most recent call last):
File "/home/developer/ros2_ws/ros2/src/planning_problem/planning_prob_pkg/planning_prob_node.py", line 102, in <module>
if __name__ == '__main__': main()
File "/home/developer/ros2_ws/ros2/src/planning_problem/planning_prob_pkg/planning_prob_node.py", line 96, in main
rclpy.spin(minimal_action_server)
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/__init__.py", line 119, in spin
executor.spin_once()
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/executors.py", line 572, in spin_once
raise handler.exception()
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/task.py", line 206, in __call__
self._handler.send(None)
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/action/server.py", line 323, in _execute_goal
execute_result = await await_or_execute(execute_callback, goal_handle)
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/executors.py", line 92, in await_or_execute
return callback(*args)
File "/home/developer/ros2_ws/ros2/src/planning_problem/planning_prob_pkg/planning_prob_node.py", line 59, in execute_callback
self._publisher.publish(msg)
File "/opt/ros/crystal/lib/python3.6/site-packages/rclpy/publisher.py", line 28, in publish
_rclpy.rclpy_publish(self.publisher_handle, msg)
ValueError: PyCapsule_GetPointer called with invalid PyCapsule object
I`m not very good with ros python interface
But at first glance, is it you create the publisher as bool type
and you are sending it as a string?
This is my first attempt to extract tweets using twitter api and tweepy. When I execute my code it keep printing 401 every time in a new line. What am I doing wrong is I am not able to figure out. Any help is appreciated.
import tweepy
import json
access_token = ""
access_token_secret = ""
consumer_key = ""
consumer_secret = ""
auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
class MyStreamListener(tweepy.StreamListener):
def __init__(self, api=None):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
self.file = open("tweets.txt", "w")
def on_status(self, status):
tweet = status._json
self.file.write( json.dumps(tweet) + '\n' )
self.num_tweets += 1
if self.num_tweets < 100:
return True
else:
return False
self.file.close()
def on_error(self, status):
print(status)
l = MyStreamListener()
stream=tweepy.Stream(auth,l)
stream.filter()
tweets_data_path = 'tweets.txt'
tweets_file = open(tweets_data_path, "r")
tweets_data = []
for line in tweets_file:
tweet = json.loads(line)
tweets_data.append(tweet)
tweets_file.close()
print(tweets_data[0].keys())
Go to your twitter account settings and change timezone to that as of your computer. Then, go to twitter app settings and generate new consumer key and new access token. These newly generated keys and tokens you should use to avoid 401 error.
Soooo I have been working on a script I took from ArcGIS Blueprints:
http://file.allitebooks.com/20151230/ArcGIS%20Blueprints.pdf
It should convert geolocated tweets into a geodatabase. I have the Twitter Streaming API already operational, and been playing with different ways to extract x/y, but keep coming back to this script, every so often, hoping I can get it running with no luck. I am stuck on a "List Index Out of Range" error. If anyone is gracious enough to offer some ideas on how I can get by this error I will be forever grateful. If nothing else this endeavor has exploited my shortcomings with Python and Arcpy, and hopefully it will round me out in the long run. For right now, I sure would like to get some mileage out of this script and the work Ive invested into it. Thank you!
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import arcpy
import sys
import time
consumer_key = 'xxx'
consumer_secret = 'xxx'
token_key = 'xxx'
token_secret = 'xxx'
class StdOutListener(StreamListener):
def __init__(self, start_time, featureClass, time_limit):
super(StdOutListener, self).__init__()
self.time = start_time
self.limit = time_limit
self.featureClass = featureClass
def on_status(self, status):
while (time.time() - self.time) < self.limit:
if status.geo is not None:
dictCoords = status.geo
listCoords = dictCoords['coordinates']
latitude = listCoords[0]
longitude = listCoords[1]
cursor =arcpy.da.InsertCursor(self.featureClass,"SHAPE#XY"))
cursor.insertRow([(longitude,latitude)])
print(str(listCoords[0]) + "," + str(listCoords[1]))
return True
else:
print "No coordinates found"
return True
start_time = time.time()
arcpy.env.workspace = "c:\ArcGIS_Blueprint_Python\data\Twitter\TweetInformation.gdb" "
def main():
try: #new
featureClass = sys.argv[1]
monitorTime = sys.argv[2]
monitorTime = monitorTime * 3600
sr = arcpy.SpatialReference(4326)
arcpy.env.overwriteOutput = True
arcpy.CreateFeatureclass_management(arcpy.env.workspace,
featureClass, "POINT", spatial_reference=sr)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(token_key, token_secret)
stream = Stream(auth, StdOutListener(start_time, featureClass,
time_limit=monitorTime)) #172800
stream.filter(track=['car'])
except Exception as e:
print(e.message)
if __name__ == '__main__':
main()
This code was working fine few moments ago but now its not working?
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import time
import random
consumer_key = ""
consumer_secret = ""
access_token_key = ""
access_token_secret = ""
Coords = dict()
Place = dict()
PlaceCoords = dict()
XY = []
class StdOutListener(StreamListener):
""" A listener handles tweets that are the received from the stream.
This is a basic listener that inserts tweets into MySQLdb.
"""
def on_status(self, status):
#print "Tweet Text: ",status.text
text = status.text
#print "Time Stamp: ",status.created_at
try:
Coords.update(status.coordinates)
XY = (Coords.get('coordinates')) #Place the coordinates values into a list 'XY'
#print "X: ", XY[0]
#print "Y: ", XY[1]
except:
#Often times users opt into 'place' which is neighborhood size polygon
#Calculate center of polygon
Place.update(status.place)
PlaceCoords.update(Place['bounding_box'])
Box = PlaceCoords['coordinates'][0]
XY = [(Box[0][0] + Box[2][0])/2, (Box[0][1] + Box[2][1])/2]
#print "X: ", XY[0]
#print "Y: ", XY[1]
pass
# Comment out next 4 lines to avoid MySQLdb to simply read stream at console
#print {"status_id":status.id_str,"timestamp":status.created_at,"location X":XY[0],"location Y":XY[1],"text":text}
print status.id_str,status.created_at,XY[0],XY[1],text
def main():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token_key, access_token_secret)
stream = Stream(auth, l, timeout=30)
#sleep
nsecs = 2
#Only records 'locations' OR 'tracks', NOT 'tracks (keywords) with locations'
while True:
try:
# Call tweepy's userstream method
#stream.filter(locations=[-122.75,36.8,-121.75,37.8,-74,40,-73,41],languages=['es','tr','ko','fr','ru','de','ja','it','pt'], async=False)##These coordinates are approximate bounding box around USA
stream.filter(locations=[-180,-90,180,90],async=False)
#stream.filter()
#stream.filter(track=['obama'])## This will feed the stream all mentions of 'keyword'
break
except Exception, e:
print Exception , e
# Abnormal exit: Reconnect
#nsecs=random.randint(30)
print "Reconnecting ",nsecs
time.sleep(nsecs)
if __name__ == '__main__':
main()
Is there any other way to collect streaming data based on locations?
No, using the locations parameter is the only way. However, I would strongly advise against your catch-all except statement in the StdOutListener. It is most likely catching a different type of error than the one you expect, masking a problem.
We have a Windows based SPSS server say 10.20.30.40. We would like to kick off a SPSS Production job from another server 10.20.30.50.
Can we kick off the job using a batch file?
1.Create an SPJ file in production.
2.make a bat file to run spj
"C:\Program Files\IBM\SPSS\Statistics\21\stats.exe" -production "K:\Meinzer\Production\SPJ\DashBoardInsert.spj"
create a 'scheduled task' in windows.
The real issue is getting your output from the job. for that, i use python.
I use syntax like this
begin program.
AlertDays=4
Files=['k:\meinzer/Production\dashboarddatasets/aod_network_report.sps',
'k:\meinzer/Production\push/aod_network_reportpush.sps',
'k:\meinzer/Production\pushproduction/aod_network_reportpushP.sps']
end program.
insert file='k:/meinzer/production/ps/errorTestPickles.sps'.
to trigger this
*still needs error info passed.
set mprint=off /printback=on.
begin program.
#test files to observe - uncomment out 8 or 9
#Files=['k:\meinzer/Production\dashboarddatasets/test.sps']
#Files=['k:\meinzer/Production\dashboarddatasets/testfail.sps']
#Files=['k:\meinzer/Production\dashboarddatasets/clinfo.sps']
#Files=['k:\meinzer/Production\dashboarddatasets/CSOC_Consecutive_High_Level_Svcs.sps']
import shutil
import spss
import re, os, pickle
from datetime import datetime
def main(Files):
"""The parser and processor for Syntax Error Reporting """
try:
for FilePath in Files:
Start = datetime.now().replace( microsecond=0)
DBname, init_Syntax = init_Vars(FilePath)
cmds = init_cmds(init_Syntax)
cmd=''
cmd2=''
cmd3=''
try:
for cmd in cmds:
cmd=cmd.replace('\r\n','\n ')
cmd=cmd.replace('\t',' ')
print cmd
spss.Submit(cmd)
cmd3=cmd2
cmd2=cmd
# cmd, cmd2, cmd3=run_cmds(cmd,cmd2,cmd3,cmds)
Finish = datetime.now().replace( microsecond=0)
spss_Output(DBname)
SavedNewname=check_saved_new_name(DBname)
if SavedNewname==1:
send_result(DBname,'Failure',Start,Finish,0,cmd,cmd2,cmd3)
break
if SavedNewname==0:
send_result(DBname,'Success',Start,Finish,1,AlertDays)
except Exception,e:
Finish = datetime.now().replace( microsecond=0)
errorLevel, errorMsg = get_spss_error(e)
send_result(DBname,"Failure in code",Start,Finish,0,AlertDays,cmd,cmd2,cmd3,errorLevel, errorMsg )
spss_Output(DBname)
break
except IOError:
print "can't open file or difficulty initializing comands from spss"
send_result('Can not open File %s' % DBname,Start,Finish)
spss_Output(DBname)
def init_Vars(FilePath):
FilePath=FilePath.encode('string-escape')
#FilePath= map(os.path.normpath, FilePath)
FilePath=FilePath.replace('\\','/')
FilePath=FilePath.replace('/x07','/a')
FilePath=FilePath.replace('//','/')
FilePath=FilePath.replace('/x08','/b')
FilePath=FilePath.replace('/x0b','/v')
FilePath=FilePath.replace('/x0c','/v')
print 'this is the file path..................... '+FilePath
DBname = os.path.split(os.path.normpath(FilePath))[-1]
#if '\\' in FilePath:
# DBname=FilePath.rpartition('\\')[-1]
#if '/' in FilePath:
# DBname=FilePath.rpartition('/')[-1]
init_Syntax=FilePath
OutputClose="output close name=%s." % DBname
OutputNew="output new name=%s." % DBname
spss.Submit(OutputClose)
spss.Submit(OutputNew)
return (DBname, init_Syntax)
def init_cmds(init_Syntax):
with open(init_Syntax,'rb') as f:
BOM_UTF8 = "\xef\xbb\xbf"
code = f.read().lstrip(BOM_UTF8)
#r = re.compile('(?<=\.)\s*?^\s*',re.M)
r = re.compile('(?<=\.)\s*?^\s*|\s*\Z|\A\s*',re.M)
cmds = r.split(code)
#cmds = re.split("(?<=\\.)%s[ \t]*" % os.linesep, code, flags=re.M)
#cmds = re.split(r'(?<=\.)[ \t]*%s' % os.linesep, code, flags=re.M)
cmds = [cmdx.lstrip() for cmdx in cmds if not cmdx.startswith("*")]
return cmds
def run_cmds(cmd,cmd2,cmd3,cmds):
for cmd in cmds:
cmd=cmd.replace('\r\n','\n ')
cmd=cmd.replace('\t',' ')
print cmd
spss.Submit(cmd)
cmd3=cmd2
cmd2=cmd
return (cmd, cmd2, cmd3)
def send_result(DBname,result,Start,Finish,status,AlertDays,cmd='',cmd2='',cmd3='',errorLevel='', errorMsg=''):
""" """
print result + ' was sent for '+DBname
FinishText = Finish.strftime("%m-%d-%y %H:%M")
StartText = Start.strftime("%m-%d-%y %H:%M")
Runtimex = str(Finish-Start)[0:7]
error_result="""%s %s
Start Finish Runtime Hrs:Min:Sec
%s %s %s """ % (DBname,result,StartText,FinishText,Runtimex)
error_result_email="""%s <br>
%s <br> Runtime %s <br>\n""" % (result,DBname,Runtimex)
with open("k:/meinzer/production/output/Error Log.txt", "r+") as myfile:
old=myfile.read()
myfile.seek(0)
if status==1:
myfile.write(error_result+"\n\n"+ old)
if status==0:
myfile.write(error_result+'\n'+'This was the problem\n'+errorLevel+" "+ errorMsg+'\n'+cmd3+'\n'+cmd2+'\n'+cmd+"\n\n"+ old)
# with open("k:/meinzer/production/output/email Log.txt", "r+") as emailtext:
# olde=emailtext.read()
# emailtext.seek(0)
# emailtext.write(error_result_email+ olde)
with open("k:/meinzer/production/output/ErrorCSV.txt", "r+") as ErrorCSV:
oldcsv=ErrorCSV.read()
ErrorCSV.seek(0)
ErrorCSV.write(DBname+','+str(status)+','+FinishText+",0"+','+str(AlertDays)+"\n"+ oldcsv)
def check_saved_new_name(DBname):
""" check saved new name"""
with open("k:/meinzer/production/output/text/"+DBname+".txt", "r") as searchfile:
if 'Warning # 5334' in open("k:/meinzer/production/output/text/"+DBname+".txt", "r").read():
SavedNewname=True
else:
SavedNewname=False
return SavedNewname
def get_spss_error(e):
print 'Error', e
errorLevel=str(spss.GetLastErrorLevel())
errorMsg=spss.GetLastErrorMessage()
return (errorLevel, errorMsg)
def spss_Output(DBname):
""" """
outputtext="output export /text documentfile='k:/meinzer/production/output/text/%s.txt'." % DBname
outputspv="output save outfile='k:/meinzer/production/output/%s.spv'." % DBname
spss.Submit(outputspv)
spss.Submit(outputtext)
main(Files)
end program.