I want to analyze tweets and ı got streamming tweets.... directly I want to streaming and saving json format of tweets.
how to save in json format while during streaming tweets via twitter api, I wrote and get stream tweets but ı didn't save json file while streaming.
class TwitteStreamer():
def stream_tweet(self,hashtag_list):
stream = tweepy.Stream(
twitter_key.consumer_key, twitter_key.consumer_secret,
twitter_key.access_token, twitter_key.access_token_secret
)
stream.filter(track=hashtag_list, languages=["en"])
this is custom listener
class TweetPrinter(tweepy.Stream):
file = open('tweets.json', 'a')
def on_status(self, status):
print(status.text)
def on_data(self,tweet):
json_data = json.loads(data)
file.write(str(json_data))
def on_error(self, status_code):
print(status_code)
time.sleep(120)
return False # To continue listening
def on_timeout(self):
time.sleep(120)
return True # To continue listening
class Printer():
def print_tweets(self, hashtag_list):
printer = TweetPrinter(
twitter_key.consumer_key, twitter_key.consumer_secret,
twitter_key.access_token, twitter_key.access_token_secret
)
#printer.filter(track=hashtag_list)
printer.filter(track=hashtag_list ,languages=["en"])
hashtag_list = ["btc","bitcoin","Bitcoin","#btc","#Bitcoin","BTC","#BTC"]
fetched_tweets_filename = "tweets.json"
twitter_streamer = TwitteStreamer()
twitter_streamer.stream_tweet(hashtag_list)
printer = Printer()
printer.print_tweets(hashtag_list)
Related
This is my first attempt to extract tweets using twitter api and tweepy. When I execute my code it keep printing 401 every time in a new line. What am I doing wrong is I am not able to figure out. Any help is appreciated.
import tweepy
import json
access_token = ""
access_token_secret = ""
consumer_key = ""
consumer_secret = ""
auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
class MyStreamListener(tweepy.StreamListener):
def __init__(self, api=None):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
self.file = open("tweets.txt", "w")
def on_status(self, status):
tweet = status._json
self.file.write( json.dumps(tweet) + '\n' )
self.num_tweets += 1
if self.num_tweets < 100:
return True
else:
return False
self.file.close()
def on_error(self, status):
print(status)
l = MyStreamListener()
stream=tweepy.Stream(auth,l)
stream.filter()
tweets_data_path = 'tweets.txt'
tweets_file = open(tweets_data_path, "r")
tweets_data = []
for line in tweets_file:
tweet = json.loads(line)
tweets_data.append(tweet)
tweets_file.close()
print(tweets_data[0].keys())
Go to your twitter account settings and change timezone to that as of your computer. Then, go to twitter app settings and generate new consumer key and new access token. These newly generated keys and tokens you should use to avoid 401 error.
I tried to render a variable('predictions') in URL(/predict) on url('/hello'). I am beginner in the web development. If someone knows , please help.
app = Flask(__name__)
#app.route('/predict', methods=['POST'])
def apicall(responses = None):
test_json = request.get_json()
test = pd.read_json(test_json, orient='records')
query_df = pd.DataFrame(test)
clf = 'kmeans__model.pkl'
print("Loading the model...")
lin_reg_model = None
with open(clf,'rb') as f:
lin_reg_model = joblib.load(f)
# lin_reg_model = joblib.load('/home/q/new_project/models/kmeans_model.pkl')
print("The model has been loaded...doing predictions now...")
predictions = lin_reg_model.predict(test)
print(predictions)
prediction_series = list(pd.Series(predictions))
final_predictions = pd.DataFrame(list(zip(prediction_series)))
responses = jsonify(predictions=final_predictions.to_json(orient="records"))
responses.status_code = 200
return (responses)
#app.route('/hello')
def hello():
**What should be used here to render *predictions*?**
return 'Hello, World '
You can return to template and show in html if you like. Create templates folder and create html file.
import requests
def hello():
response = requests.get(url_for('apicall'))
return render_template('<yourtemplate.html>', predictions=response.json())
HTML
{{ predictions }}
Note: Please follow this link for any problem with thread and deployment Handling multiple requests in Flask
I'm new to Rails.
I have a method that downloads a file from google drive and saves it in the local disc. When the file is downloaded, the console returns nil, but the file is in the folder.
I need to use this file in my controller, but if the download method is returning nil I can't pass it around as an object.
download method:
def download
found = google_files.select { |f| f.id == file_id }
file_title = found.first.title
file = session.file_by_title(file_title)
path = File.join(Backup.base_directory, file_title)
file.download_to_file("#{path}")
end
Controller:
def create
# file_id = params.fetch(:file_id)
file_id = "0Byyflt8z3jarbm5DZGNNVXZSWjg"
#backup = DiscourseDownloadFromDrive::DriveDownloader.new(file_id).download
end
console output after executing the download method:
[...]
Writing chunk (1397 bytes)
Writing chunk (1397 bytes)
Writing chunk (1397 bytes)
Writing chunk (619 bytes)
Success - nil
=> nil
[4] pry(main)>
Logger:
Rails.logger.debug(">>> #BACKUP >>>: #{#backup.inspect}")
D, [2017-09-07T20:21:24.835450 #7755] DEBUG -- : >>> #BACKUP >>>: nil
Any hint on how to proceed with this would be very much appreciated!
Your download method always returns nothing but nil. That's because the gem's download_to_file always returns nil.
You shoud change your download method for it to return something, that you can use to get the file. I think this method should return the path to the downloaded file.
def download
found = google_files.select { |f| f.id == file_id }
file_title = found.first.title
file = session.file_by_title(file_title)
path = File.join(Backup.base_directory, file_title)
file.download_to_file("#{path}")
path
end
Now you can use it in the controller:
def create
# file_id = params.fetch(:file_id)
file_id = "0Byyflt8z3jarbm5DZGNNVXZSWjg"
file_path = DiscourseDownloadFromDrive::DriveDownloader.new(file_id).download
#backup = File.open(file_path) # do whatever you want with the file, since now you know how to get it
end
I am using groovy script for automated deployment in jenkins.
In have a xml file, I want to read specific xml tag value from that xml file and assign it's value to a variable. Letter I am using that variable to write excel file. Below is the script. After executing the build it seems that I am not able to access the xml tag value. It seems the code inside of each function is not working. Can anyone please help me on this
def Build_Name= ""
def Build_Number= ""
def CR_Number=""
def Build_Status= ""
def Build_Failure_Reason=""
def Build_Date=""
Build_Number= "1201"
CR_Number="123"
Build_Status= "SUCCESS"
Build_Failure_Reason="NA"
Build_Date=""
boolean fileSuccessfullyDeleted = new File("/export/home/webm/.jenkins/Build_Report.csv").delete()
f = new File("/export/home/webm/.jenkins/Build_Report.csv")
def Job_Result_file = new File("/export/home/webm/.jenkins/global-build-stats/jobresults/Temp_Job_Result.xml")
//Parse it with XmlSlurper
def xml = new XmlSlurper().parse(Job_Result_file)
xml.list.jbr.each { sd ->
Build_Name= sd.n[0].value
f.append("\r\n"+Build_Name+","+Build_Number+","+CR_Number+","+Build_Status+","+Build_Failure_Reason+","+Build_Date+"")
}
EDIT: based on OP added as answer.
Below is the xml available in xml file and I want to read the value of tag and want to assign that value to a variable. In below xml tag n has value Send_Deployment_Request.
Xml:
<list>
<jbr plugin="global-build-stats#1.4">
<r>SUCCESS</r>
<n>Send_Deployment_Request</n>
<nb>187</nb>
<d>
<time>1502879887698</time>
<timezone></timezone>
</d>
<du>973</du>
<nn>master</nn>
<un></un>
</jbr>
<jbr plugin="global-build-stats#1.4">
<r>SUCCESS</r>
<n>Deploy</n>
<nb>187</nb>
<d>
<time>1502879887698</time>
<timezone></timezone>
</d>
<du>973</du>
<nn>master</nn>
<un></un>
</jbr>
</list>
Dear Rao,
Please see my code below.
//Delete if Build_Report fiile already exist
boolean fileSuccessfullyDeleted = new File("/export/home/webmadm/.jenkins/Build_Report.csv").delete()
//Craete new csv file
f = new File("/export/home/webmadm/.jenkins/Build_Report.csv")
def Job_Result_file = new File("/export/home/webmadm/.jenkins/global-build-stats/jobresults/Temp_Job_Result.xml")
def xml = new XmlSlurper().parse(Job_Result_file)
//Change the element name if needed
def Temp_Build_Name= "n"
def Temp_Build_Number= "nb"
def CR_Number=""
def Temp_Build_Status= "r"
def Build_Failure_Reason=""
def Temp_Build_Date="time"
def Temp_Build_Executor="un"
def Build_Name= ""
def Build_Number= ""
def Build_Status= ""
def Build_Date=""
def Build_Executor=""
//Get all the values of Tag - elementToFind
def Build_Name_List= xml.'**'.findAll{'it.name() == Temp_Build_Name'}
def Build_Number_List= xml.'**'.findAll{'it.name() == Temp_Build_Number'}
def Build_Status_List= xml.'**'.findAll{'it.name() == Temp_Build_Status'}
def Build_Date_List= xml.'**'.findAll{'it.name() == Temp_Build_Date'}
def Build_Executor_List= xml.'**'.findAll{'it.name() == Temp_Build_Executor'}
for(int i = 0; i<2; i++)
{
Build_Name= Build_Name_List[i].text()
Build_Number= Build_Number_List[i].text()
Build_Status= Build_Status_List[i].text()
Build_Date= Build_Date_List[i].text()
Build_Executor= Build_Executor_List[i].text()
f.append("\r\n"+Build_Name+","+Build_Number+","+CR_Number+","+Build_Status+","+Build_Failure_Reason+","+Build_Date+"")
}
I am trying to write .csv file. If you see f.append as per that in csc we have Build_Name value , Build_Number value and so on.
But actul result is different, Build_Name also contain the Build_Number, Build_Status Build_Date. Seems .findAll method reading entire tag from xml.
Can't we use .each method as we use it to update the xml node value in xml file
?
Here is what you would need and comments in-line:
def xml = new XmlSlurper().parse(Job_Result_file)
//Change the element name if needed
def elementToFind = 'n'
//Get all the values of Tag - elementToFind
def result = xml.'**'.findAll{it.name() == elementToFind}
println result
//Get the first item from the result
def requiredValue = result[0].text()
println "User required value : $requiredValue"
//Test if the value is as expected
assert 'Send_Deployment_Request' == requiredValue
You can quickly see it online demo
class SampleTest(APITestCase):
def setUp(self):
self.id = 1
self.api_url = 'api/check_customers'
self.token ='##############'
def test_check_customer(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer ' + self.token)
response = self.client.post(self.api_url, data={'id': self.id})
self.assertEqual(response.status_code, status.HTTP_200_OK)
When I test this code I get the error message which I have set for checking the emptyness of parameter like
{'message': 'id is empty', 'status': 'failed'}
and the way I check this is in views
class Customer(APIView):
def post(self, request, *args, **kwargs):
if "id" not in request.data:
content = {"status": "failed",
"message": "id is empty"}
return Response(content, status=STATUS.HTTP_400_BAD_REQUEST)
I am using DRF 3.3.0 and Django 1.9
response = self.client.post(self.api_url, data={'id': self.id}, format='json')
This works Fine. Default format type is multipart that has to be json while passing dictionary