Is there a Bitbucket API to search if a repository variable is defined in all of my workspace's repos? - bitbucket

Instead of defining a Bitbucket Cloud workspace variable that can be used by all the repos in the workspace, someone defined it in each repo, but not in all of them, of the workspace. Now I want to remove the variable in the individual repos, and define it in the workspace.
Is there a Bitbucket API that would do this pseudo-code?
def bb = Bitbucket.getInstance()
String workspace = "MyWorkspace"
String myVariable = "NEXUS_USER"
List<Repository> reposInWorkspace = bb.getWorkspace(workspace).getAllReposInWorkspace()
reposInWorkspace.each { repo ->
if (repo.hasVariable(myVariable)) {
println repo.name
}
}

I put a Bitbucket support ticket, and a sharp Atlassian support person gave me this Python3 script
from requests import Session
from time import sleep
username = 'your_username_not_email'
password = 'app_pw_not_bb_user_pw'
workspace = 'your_workspace'
variable_name = 'your_variable'
URL = f'https://api.bitbucket.org/2.0/repositories/{workspace}'
session = Session()
session.auth = (username, password)
def get_repos(page=None):
while True:
params = {'page': page, 'pagelen': 100}
r = session.get(URL, params=params)
while r.status_code == 429:
print("Hit the API rate limit. Sleeping for 10 sec...")
sleep(10)
print("Resuming...")
r = session.get(URL, params=params)
r_data = r.json()
for repo in r_data.get('values'):
yield repo.get('slug')
if not r_data.get('next'):
return
if page is None:
page = 1
page += 1
def get_variables(repo, page=None):
while True:
params = {'page': page, 'pagelen': 100}
r = session.get(f'{URL}/{repo}/pipelines_config/variables/', params=params)
while r.status_code == 429:
print("Hit the API rate limit. Sleeping for 10 sec...")
sleep(10)
print("Resuming...")
r = session.get(URL, params=params)
r_data = r.json()
for var in r_data.get('values'):
yield var.get('key')
if not r_data.get('next'):
return
if page is None:
page = 1
page += 1
def has_variable(var):
if var == variable_name:
return True
def main():
for repo in get_repos():
for var in get_variables(repo):
if has_variable(var):
print(f'{repo}')
if __name__ == '__main__':
main()

Related

Lua variable not updated in wrk

I need to run a script in wrk to test my endpoint. My script is as followed:
thread_counter = 0
local thread_id = 1
setup = function (thread)
thread_counter = thread_counter + 1
thread_id = thread_counter
print("thread_id 1 " ..thread_id)
end
request = function()
idx = math.random(200)
print("thread_id 2 " ..thread_id)
...
end
I can see that thread_id is updated in the setup function, but not the request function. Why and how can I fix this?

Search and replace with commit in bitbucket multiple repositories

Is there a way to search and replace some string in Bitbucket git repositories under some project using Bitbucket administration tools/API/UI? The modification should be via git commit.
The only way that pops to mind is building some script which will go over all repositories.
I had to do just such an operation. When we migrated from svn to git, we had to change the scm developerConnection in our pom.xml:
<developerConnection>scm:git:git#github.com:codehaus-plexus/plexus-interpolation.git</developerConnection>
in all our git repos from the svn developer connection to the git one. Here is the script I used:
#!/usr/bin/python
import stashy
import os
import sys
import urllib2
import json
import base64
import getpass
from git import Repo
import xml.etree.ElementTree as ET
from xml.etree.ElementTree import Element
from lxml import etree
import itertools
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
bitbucketBaseUrl = "https://bitbucket.company.com"
bitbucketUserName = "admin"
class BitBucketRepo:
def __init__(self, name, repoUrl):
self.name = name
self.repoUrl = repoUrl
class CommentedTreeBuilder(ET.TreeBuilder):
def __init__(self, *args, **kwargs):
super(CommentedTreeBuilder, self).__init__(*args, **kwargs)
def comment(self, data):
self.start(ET.Comment, {})
self.data(data)
self.end(ET.Comment)
def insert(originalfile, string):
with open(originalfile, 'r') as f:
with open('pom.xml.bak', 'w') as f2:
f2.write(string + "\n")
f2.write(f.read())
os.rename('pom.xml.bak', originalfile)
def validateScriptParameters():
if len(sys.argv) != 3:
sys.exit("Usage: {} [Bit Bucket Module Project key, e.g. mf for Modules - Framework] [Bit Bucket admin password]".format(
os.path.basename(sys.argv[0])))
def cloneRepo(repository):
logging.info("Cloning repo [{}]".format(repository.repoUrl))
repo = Repo.clone_from(repository.repoUrl, repository.name)
return repo
def updatePomFile(repository):
resultCode = 1
ET.register_namespace('xsi', "http://www.w3.org/2001/XMLSchema-instance")
ET.register_namespace('', "http://maven.apache.org/POM/4.0.0")
cparser = ET.XMLParser(target = CommentedTreeBuilder())
if os.path.isfile(os.getcwd() + "/" + repository.name + "/pom.xml"):
tree = ET.parse(repository.name + "/pom.xml", parser=cparser)
root = tree.getroot()
ns = {'nodes': 'http://maven.apache.org/POM/4.0.0'}
for scm in root.findall('nodes:scm', ns):
if scm is not None:
developerConnection = scm.find('nodes:developerConnection', ns)
scm.remove(developerConnection)
newDeveloperConnectionElm = Element("developerConnection")
newDeveloperConnectionElm.tail = "\n\t"
newDeveloperConnectionElm.text = str("scm:git:" + repository.repoUrl)
scm.append(newDeveloperConnectionElm)
resultCode = 0
else:
resultCode = 1
if resultCode == 0:
logging.info("Updating repository: " + repository.name)
tree.write(repository.name + "/pom.xml")
return resultCode
def updateRepo(bitbucket, projectKey):
xmlVersion = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
repoList = bitbucket.projects[projectKey].repos
for repoItem in repoList:
cloneUrl = repoItem['links']['clone'][1]['href']
if cloneUrl.startswith("http"):
cloneUrl = repoItem['links']['clone'][0]['href']
if not cloneUrl.startswith("ssh"):
logging.error("Unable to retrieve valid clone url [{}], exiting...".format(cloneUrl))
sys.exit(1)
repository = BitBucketRepo(repoItem['name'], cloneUrl)
clonedRepo = cloneRepo(repository)
resultCode = updatePomFile(repository)
if resultCode == 0:
insert(repository.name + "/pom.xml", xmlVersion)
clonedRepo.index.add([os.getcwd() + "/" + repository.name + "/pom.xml"])
clonedRepo.index.commit("CM-8991: Updating pom.xml to use git connection string instead of svn")
clonedRepo.remotes.origin.push()
validateScriptParameters()
logging.info('Bit Bucket URL [' + bitbucketBaseUrl + ']')
logging.info('User name [' + bitbucketUserName + ']')
projectKey = sys.argv[1]
bitbucketPassword = sys.argv[2]
bitbucket = stashy.connect(bitbucketBaseUrl, bitbucketUserName, bitbucketPassword)
logging.info("Module project key: [{}]".format(projectKey))
updateRepo(bitbucket, projectKey)
Note that it requires the 'stashy' python library, and is written for python2.

Basic Twitter Data Mining Causing Problem

This is my first attempt to extract tweets using twitter api and tweepy. When I execute my code it keep printing 401 every time in a new line. What am I doing wrong is I am not able to figure out. Any help is appreciated.
import tweepy
import json
access_token = ""
access_token_secret = ""
consumer_key = ""
consumer_secret = ""
auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
class MyStreamListener(tweepy.StreamListener):
def __init__(self, api=None):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
self.file = open("tweets.txt", "w")
def on_status(self, status):
tweet = status._json
self.file.write( json.dumps(tweet) + '\n' )
self.num_tweets += 1
if self.num_tweets < 100:
return True
else:
return False
self.file.close()
def on_error(self, status):
print(status)
l = MyStreamListener()
stream=tweepy.Stream(auth,l)
stream.filter()
tweets_data_path = 'tweets.txt'
tweets_file = open(tweets_data_path, "r")
tweets_data = []
for line in tweets_file:
tweet = json.loads(line)
tweets_data.append(tweet)
tweets_file.close()
print(tweets_data[0].keys())
Go to your twitter account settings and change timezone to that as of your computer. Then, go to twitter app settings and generate new consumer key and new access token. These newly generated keys and tokens you should use to avoid 401 error.

Access variable and render it in different URL

I tried to render a variable('predictions') in URL(/predict) on url('/hello'). I am beginner in the web development. If someone knows , please help.
app = Flask(__name__)
#app.route('/predict', methods=['POST'])
def apicall(responses = None):
test_json = request.get_json()
test = pd.read_json(test_json, orient='records')
query_df = pd.DataFrame(test)
clf = 'kmeans__model.pkl'
print("Loading the model...")
lin_reg_model = None
with open(clf,'rb') as f:
lin_reg_model = joblib.load(f)
# lin_reg_model = joblib.load('/home/q/new_project/models/kmeans_model.pkl')
print("The model has been loaded...doing predictions now...")
predictions = lin_reg_model.predict(test)
print(predictions)
prediction_series = list(pd.Series(predictions))
final_predictions = pd.DataFrame(list(zip(prediction_series)))
responses = jsonify(predictions=final_predictions.to_json(orient="records"))
responses.status_code = 200
return (responses)
#app.route('/hello')
def hello():
**What should be used here to render *predictions*?**
return 'Hello, World '
You can return to template and show in html if you like. Create templates folder and create html file.
import requests
def hello():
response = requests.get(url_for('apicall'))
return render_template('<yourtemplate.html>', predictions=response.json())
HTML
{{ predictions }}
Note: Please follow this link for any problem with thread and deployment Handling multiple requests in Flask

Lua script error when run from stackexchange.redis

We have a lengthy lua script. Please find it below after the problem description.
The logic of the script is to return true/false if all the 3 parameters match (UserName, UserCategory, UserCategoryItem)
The first part of the script splits the RedisKeys to fetch the UserName, UserCategory, UserCategoryItem. There is also a random function to test if ‘functions’ works in the Redis lua script. It works.
The second part of the script compares this values against the values in the Database. The values are hard-coded in the script. If all the values match it returns true else it returns false.
The two scripts run perfect when ran individually. But they throw an error when run together.
ISSUE/PROBLEM
All the three scripts run perfectly when run in an IDE. I use the SciTE IDE (an IDE for Lua for Windows envrionment)
The script creates an issue when run from the Stackexchange.Redis client.
The first two parts of the script (Part One and Part Two) run perfectly from the Stackexchange.Redis. It is only when they are combined in the third script (Part Three) does it throw an error.
As the error is generic in nature hence I am not able to further investigate the issue.
I have divided the script in Three parts. The final script combines the first two scripts.
var redis = ConnectionMultiplexer.Connect("localhost");
var db = redis.GetDatabase();
//PART ONE OF THE SCRIPT
var doesExist1 = (string)db.ScriptEvaluate(#"
local function isDateInRange(DateStart, DateEnd, GivenDate)
if (GivenDate > DateStart and GivenDate < DateEnd)
then
return(true)
else
return(false)
end
end
--Fetch User's Category's StartDate and EndDate
local function IsCategoryDateValid()
local givenDate, DateStart, DateEnd
GivenDate = '2017-01-09'
StartDate = '2015-01-01'
EndDate = '2018-01-01'
local isDateValid
isDateValid = isDateInRange(StartDate, EndDate, GivenDate)
return(isDateValid)
end
-- Pass User, UserCategory and UserCategoryItem as parameters
local userNameKey = 'UserDetails:DummyName'
local userCategoryKey = 'UserCategories:Book'
local userCategoryItemKey = 'UserCategoryItems:Harry-Potter'
local userNameKeySplit = {}
local a = 1
for i in string.gmatch(userNameKey, '([^'..':'..']+)') do
userNameKeySplit[a] = i
a = a + 1
end
local userName = userNameKeySplit[2]
local userCategoryKeySplit = {}
local b = 1
for j in string.gmatch(userCategoryKey, '([^'..':'..']+)') do
userCategoryKeySplit[b] = j
b = b + 1
end
local userCategory = userCategoryKeySplit[2]
local userCategoryItemKeySplit = {}
local c = 1
for k in string.gmatch(userCategoryItemKey, '([^'..':'..']+)') do
userCategoryItemKeySplit[c] = k
c = c + 1
end
local userCategoryItem = userCategoryItemKeySplit[2]
return(userName..' '..userCategory..' '..userCategoryItem)
",
new RedisKey[] { "UserCategoryItemNames:" + userModel.UserId });
//PART TWO OF THE SCRIPT
var doesExist2 = (bool)db.ScriptEvaluate(#"
local userName = 'DummyName'
local userCategory = 'Book'
local userCategoryItem = 'Harry-Potter'
-- Fetch Users, UserCategories and UserCategoryItems from the Redis DB
local userData = {'DummyName', 'User1', 'User2', 'User3'}
local userCategoryData = {'Book', 'Movie', 'Journals'}
local userCategoryItemData = {'Hardy-Boys', 'Harry-Potter', 'Shawshank-Redemption', 'The-Terminal'}
local msg
-- Loop through the fetched values and compare them with parameters; if all the parameters are found return true else return false
for i=1,#userData,1
do
if(userData[i] == userName)
then
for i=1,#userCategoryData,1
do
if(userCategoryData[i] == userCategory)
then
for i=1,#userCategoryItemData,1
do
if(userCategoryItemData[i] == userCategoryItem)
then
msg = true
break
else
msg = false
end
end
break
else
msg = false
end
end
break
else
msg = false
break
end
end
return(msg)
//PART ONE & TWO COMBINED OF THE SCRIPT
",
new RedisKey[] { "UserDetails:" + "DummyName", "UserCategories:" + "Book", "UserCategoryItems:" + "Harry-Potter" });
var doesExist3 = (bool)db.ScriptEvaluate(#"
local function isDateInRange(DateStart, DateEnd, GivenDate)
if (GivenDate > DateStart and GivenDate < DateEnd)
then
return(true)
else
return(false)
end
end
--Fetch User's Category's StartDate and EndDate
local function IsCategoryDateValid()
local givenDate, DateStart, DateEnd
GivenDate = '2017-01-09'
StartDate = '2015-01-01'
EndDate = '2018-01-01'
local isDateValid
isDateValid = isDateInRange(StartDate, EndDate, GivenDate)
return(isDateValid)
end
-- Pass User, UserCategory and UserCategoryItem as parameters
local userNameKey = 'UserDetails:DummyName'
local userCategoryKey = 'UserCategories:Book'
local userCategoryItemKey = 'UserCategoryItems:Harry-Potter'
local userNameKeySplit = {}
local a = 1
for i in string.gmatch(userNameKey, '([^'..':'..']+)') do
userNameKeySplit[a] = i
a = a + 1
end
local userName = userNameKeySplit[2]
local userCategoryKeySplit = {}
local b = 1
for j in string.gmatch(userCategoryKey, '([^'..':'..']+)') do
userCategoryKeySplit[b] = j
b = b + 1
end
local userCategory = userCategoryKeySplit[2]
local userCategoryItemKeySplit = {}
local c = 1
for k in string.gmatch(userCategoryItemKey, '([^'..':'..']+)') do
userCategoryItemKeySplit[c] = k
c = c + 1
end
local userCategoryItem = userCategoryItemKeySplit[2]
-- Fetch Users, UserCategories and UserCategoryItems from the Redis DB
local userData = {'DummyName', 'User1', 'User2', 'User3'}
local userCategoryData = {'Book', 'Movie', 'Journals'}
local userCategoryItemData = {'Hardy-Boys', 'Harry-Potter', 'Shawshank-Redemption', 'The-Terminal'}
local msg
-- Loop through the fetched values and compare them with parameters; if all the parameters are found return true else return false
for i=1,#userData,1
do
if(userData[i] == userName)
then
for i=1,#userCategoryData,1
do
if(userCategoryData[i] == userCategory)
then
for i=1,#userCategoryItemData,1
do
if(userCategoryItemData[i] == userCategoryItem)
then
msg = true
break
else
msg = false
end
end
break
else
msg = false
end
end
break
else
msg = false
break
end
end
return(msg)
",
new RedisKey[] { "UserDetails:" + "DummyName", "UserCategories:" + "Book", "UserCategoryItems:" + "Harry-Potter" });
Thank you for your patience and reading this much. I hope I have clarified the problem in detail. Please provide some help in solving the issue. Thank you.

Resources