|
Theo Chatzimichos |
1f0a34 |
#!/usr/bin/python3
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
1f0a34 |
# For description and usage, see the argparse options at the end of the file
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
ab6c8e |
from copy import copy
|
|
Theo Chatzimichos |
042b5b |
from pygit2.errors import GitError
|
|
Theo Chatzimichos |
1f0a34 |
import argparse
|
|
Theo Chatzimichos |
1f0a34 |
import os
|
|
Theo Chatzimichos |
042b5b |
import pygit2
|
|
Theo Chatzimichos |
4af2b2 |
import sys
|
|
Theo Chatzimichos |
1f0a34 |
import yaml
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
11d230 |
def check_open_pull_requests():
|
|
Theo Chatzimichos |
11d230 |
from github import Github
|
|
Theo Chatzimichos |
11d230 |
|
|
Theo Chatzimichos |
11d230 |
g = Github()
|
|
Theo Chatzimichos |
11d230 |
for formula, data in FORMULAS.items():
|
|
Theo Chatzimichos |
11d230 |
open_pull_requests = data.get('pending', [])
|
|
Theo Chatzimichos |
11d230 |
if open_pull_requests:
|
|
Theo Chatzimichos |
11d230 |
namespace = data.get('original_namespace', 'saltstack-formulas')
|
|
Theo Chatzimichos |
11d230 |
prefix = data.get('prefix', '')
|
|
Theo Chatzimichos |
11d230 |
org = g.get_organization(namespace)
|
|
Theo Chatzimichos |
11d230 |
for pull_request in open_pull_requests:
|
|
Theo Chatzimichos |
11d230 |
pr = int(pull_request.split('/')[-1])
|
|
Theo Chatzimichos |
11d230 |
state = org.get_repo('%s%s-formula' % (prefix, formula)).get_pull(pr).state
|
|
Theo Chatzimichos |
11d230 |
print('%s is %s' % (pull_request, state))
|
|
Theo Chatzimichos |
11d230 |
|
|
Theo Chatzimichos |
11d230 |
|
|
Theo Chatzimichos |
62825a |
def git(cmd, cwd=None):
|
|
Theo Chatzimichos |
042b5b |
# TODO migrate to pygit2
|
|
Theo Chatzimichos |
7e97fe |
|
|
Theo Chatzimichos |
7e97fe |
import subprocess
|
|
Theo Chatzimichos |
7e97fe |
|
|
Theo Chatzimichos |
62825a |
status = subprocess.call(['git'] + cmd, cwd=cwd)
|
|
Theo Chatzimichos |
7e97fe |
if status != 0:
|
|
Theo Chatzimichos |
7e97fe |
sys.exit(status)
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
042b5b |
def clone(CLONE_FROM, CLONE_BRANCH, DEST):
|
|
Theo Chatzimichos |
ce9231 |
def clone_repo():
|
|
Theo Chatzimichos |
ce9231 |
FULL_PATH = '%s/%s-formula' % (DEST, formula)
|
|
Theo Chatzimichos |
ce9231 |
if os.path.isdir(FULL_PATH):
|
|
Theo Chatzimichos |
ce9231 |
return
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
042b5b |
pygit2.clone_repository(url, FULL_PATH, bare=False)
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
e2b39c |
branch_opts = []
|
|
Theo Chatzimichos |
e2b39c |
if CLONE_BRANCH:
|
|
Theo Chatzimichos |
e2b39c |
branch_opts = ['-b', CLONE_BRANCH, '--single-branch']
|
|
Theo Chatzimichos |
e2b39c |
if CLONE_FROM:
|
|
Theo Chatzimichos |
e2b39c |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
e2b39c |
url = '%s/%s-formula' % (CLONE_FROM, formula)
|
|
Theo Chatzimichos |
ce9231 |
clone_repo()
|
|
Theo Chatzimichos |
e2b39c |
else:
|
|
Theo Chatzimichos |
e2b39c |
for formula, data in FORMULAS.items():
|
|
Theo Chatzimichos |
e2b39c |
namespace = data.get('namespace', 'saltstack-formulas')
|
|
Theo Chatzimichos |
e2b39c |
prefix = data.get('prefix', '')
|
|
Theo Chatzimichos |
e2b39c |
url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula)
|
|
Theo Chatzimichos |
ce9231 |
clone_repo()
|
|
Theo Chatzimichos |
9af0f5 |
|
|
Theo Chatzimichos |
9af0f5 |
|
|
Theo Chatzimichos |
9af0f5 |
def create_symlinks(DEST):
|
|
Theo Chatzimichos |
9af0f5 |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
16b98e |
FULL_PATH = '/srv/salt/%s' % formula
|
|
Theo Chatzimichos |
16b98e |
if not os.path.islink(FULL_PATH):
|
|
Theo Chatzimichos |
16b98e |
os.symlink('%s/%s-formula/%s' % (DEST, formula, formula), FULL_PATH)
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
655681 |
def remove_symlinks():
|
|
Theo Chatzimichos |
655681 |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
655681 |
FULL_PATH = '/srv/salt/%s' % formula
|
|
Theo Chatzimichos |
655681 |
if os.path.islink(FULL_PATH):
|
|
Theo Chatzimichos |
655681 |
os.unlink(FULL_PATH)
|
|
Theo Chatzimichos |
655681 |
|
|
Theo Chatzimichos |
655681 |
|
|
Theo Chatzimichos |
25dc6d |
def fetch_remote(remote, formula):
|
|
Theo Chatzimichos |
a900e2 |
remotecallbacks = None
|
|
Theo Chatzimichos |
a900e2 |
if not remote.url.startswith(('http://', 'https://', 'git://', 'ssh://', 'git+ssh://')):
|
|
Theo Chatzimichos |
a900e2 |
username = remote.url.split('@')[0]
|
|
Theo Chatzimichos |
a900e2 |
credentials = pygit2.KeypairFromAgent(username)
|
|
Theo Chatzimichos |
a900e2 |
remotecallbacks = pygit2.RemoteCallbacks(credentials=credentials)
|
|
Theo Chatzimichos |
25dc6d |
try:
|
|
Theo Chatzimichos |
a900e2 |
remote.fetch(callbacks=remotecallbacks)
|
|
Theo Chatzimichos |
25dc6d |
except GitError:
|
|
Theo Chatzimichos |
25dc6d |
print('%s-formula: Failed to fetch remote %s' % (formula, remote.name))
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
def add_remote(REMOTES, DEST):
|
|
Theo Chatzimichos |
c0be5f |
for remote in REMOTES:
|
|
Theo Chatzimichos |
c0be5f |
namespace = None
|
|
Theo Chatzimichos |
c0be5f |
if len(remote) == 4:
|
|
Theo Chatzimichos |
c0be5f |
namespace = remote.pop()
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
url = 'https://github.com'
|
|
Theo Chatzimichos |
c0be5f |
if len(remote) == 3:
|
|
Theo Chatzimichos |
c0be5f |
url = remote.pop()
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
prefix = ''
|
|
Theo Chatzimichos |
c0be5f |
use_prefix = False
|
|
Theo Chatzimichos |
c0be5f |
if not remote[1].startswith('no'):
|
|
Theo Chatzimichos |
c0be5f |
use_prefix = True
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
name = remote[0]
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
for formula, data in FORMULAS.items():
|
|
Theo Chatzimichos |
c0be5f |
if not namespace:
|
|
Theo Chatzimichos |
c0be5f |
namespace = data.get('namespace', 'saltstack-formulas')
|
|
Theo Chatzimichos |
c0be5f |
if use_prefix:
|
|
Theo Chatzimichos |
c0be5f |
prefix = data.get('prefix', '')
|
|
Theo Chatzimichos |
c0be5f |
if not url.endswith(':'):
|
|
Theo Chatzimichos |
c0be5f |
url += '/'
|
|
Theo Chatzimichos |
c0be5f |
full_url = '%s%s/%s%s-formula' % (url, namespace, prefix, formula)
|
|
Theo Chatzimichos |
c0be5f |
FULL_PATH = '%s/%s-formula' % (DEST, formula)
|
|
Theo Chatzimichos |
c0be5f |
repo = pygit2.Repository(FULL_PATH)
|
|
Theo Chatzimichos |
c0be5f |
try:
|
|
Theo Chatzimichos |
c0be5f |
repo.create_remote(name, full_url)
|
|
Theo Chatzimichos |
ec31ef |
except ValueError: # remote already exists
|
|
Theo Chatzimichos |
ec31ef |
continue
|
|
Theo Chatzimichos |
25dc6d |
fetch_remote(repo.remotes[name], formula)
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
def update(REMOTES, DEST):
|
|
Theo Chatzimichos |
25dc6d |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
25dc6d |
FULL_PATH = '%s/%s-formula' % (DEST, formula)
|
|
Theo Chatzimichos |
25dc6d |
repo = pygit2.Repository(FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
git(['pull', '-q'], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
if REMOTES:
|
|
Theo Chatzimichos |
25dc6d |
for remote in REMOTES:
|
|
Theo Chatzimichos |
25dc6d |
fetch_remote(repo.remotes[remote], formula)
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
def push(REMOTES, DEST):
|
|
Theo Chatzimichos |
25dc6d |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
25dc6d |
FULL_PATH = '%s/%s-formula' % (DEST, formula)
|
|
Theo Chatzimichos |
25dc6d |
repo = pygit2.Repository(FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
for remote in REMOTES:
|
|
Theo Chatzimichos |
25dc6d |
git(['push', '-qf', remote, 'master'], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
git(['push', '-qf', remote, 'master:production'], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
25dc6d |
fetch_remote(repo.remotes[remote], formula)
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
2b0379 |
def checkout_remote_and_branch(REMOTE_BRANCH, DEST):
|
|
Theo Chatzimichos |
2b0379 |
for formula in FORMULAS.keys():
|
|
Theo Chatzimichos |
2b0379 |
FULL_PATH = '%s/%s-formula' % (DEST, formula)
|
|
Theo Chatzimichos |
2b0379 |
branch = REMOTE_BRANCH.split('/')[1]
|
|
Theo Chatzimichos |
2b0379 |
git(['checkout', '-qB', branch, REMOTE_BRANCH], cwd=FULL_PATH)
|
|
Theo Chatzimichos |
2b0379 |
|
|
Theo Chatzimichos |
2b0379 |
|
|
Theo Chatzimichos |
44c07a |
with open('pillar/FORMULAS.yaml', 'r') as f:
|
|
Karol Babioch |
3016f1 |
FORMULAS_YAML = yaml.safe_load(f)
|
|
Theo Chatzimichos |
ab6c8e |
|
|
Theo Chatzimichos |
ab6c8e |
FORMULAS = copy(FORMULAS_YAML)
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
c0be5f |
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description='Loads the formulas from FORMULAS.yaml and performs one or more of the operations specified at the arguments.')
|
|
Theo Chatzimichos |
25dc6d |
parser.add_argument('-q', '--pull-requests', action='store_true', help='Prints the status of the Pull Requests that are defined in FORMULAS.yaml under "pending".')
|
|
Theo Chatzimichos |
8e8456 |
parser.add_argument('-d', '--destination', nargs=1, help='Destination absolute path of the cloned (or to-be-cloned) repositories of the formulas.')
|
|
Theo Chatzimichos |
ab6c8e |
parser.add_argument('-f', '--formulas', action='append', nargs='+', help='Specify specific formulas to operate on, instead of working with all the specified FORMULAS.yaml formulas.')
|
|
Theo Chatzimichos |
cd2f66 |
parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination".')
|
|
Theo Chatzimichos |
e2b39c |
parser.add_argument('--clone-from', nargs=1, help='Specify the git provider to clone from together with the namespace.')
|
|
Theo Chatzimichos |
e2b39c |
parser.add_argument('--clone-branch', nargs=1, help='Specify the branch to clone.')
|
|
Theo Chatzimichos |
1f0a34 |
parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.')
|
|
Theo Chatzimichos |
655681 |
parser.add_argument('--remove-symlinks', action='store_true', help='Removes all symlinks that were created in /srv/salt.')
|
|
Theo Chatzimichos |
c0be5f |
parser.add_argument('-r', '--add-remote', action='append', nargs='+', help='''Add the specified remotes on the local repositories. It can be passed multiple times.
|
|
Theo Chatzimichos |
c0be5f |
Usage: REMOTE_NAME USE_PREFIXES [GIT_PROVIDER_URL] [NAMESPACE].
|
|
Theo Chatzimichos |
c0be5f |
- REMOTE is string
|
|
Theo Chatzimichos |
c0be5f |
- USE_PREFIXES should be a string starting with "no" (for no prefix usage), or whatever else string (for prefix usage)
|
|
Theo Chatzimichos |
c0be5f |
- GIT_URL (optional) can be in the form "https://gitlab.example.com" or "git@gitlab.example.com:" (make sure you have the trailing colon). If no git provider URL is given, https://github.com will be used.
|
|
Theo Chatzimichos |
c0be5f |
- NAMESPACE (optional) is string. If no namespace is given, the one defined in FORMULAS.yaml will be used.
|
|
Theo Chatzimichos |
c0be5f |
Examples:
|
|
Theo Chatzimichos |
c0be5f |
-r forks_ro prefixes
|
|
Theo Chatzimichos |
c0be5f |
-r forks_rw prefixes git@github.com:
|
|
Theo Chatzimichos |
c0be5f |
-r mycompany no_prefixes https://gitlab.mycompany.com saltstack-formulas
|
|
Theo Chatzimichos |
c0be5f |
-r mycompany_forks no_prefixes git@gitlab.mycompany.com: saltstack-formulas''')
|
|
Theo Chatzimichos |
25dc6d |
parser.add_argument('-u', '--update', nargs='*', help='Switch to origin/master and git pull. Optionally it can accept a list of remotes as arguments, that will be fetched.')
|
|
Theo Chatzimichos |
25dc6d |
parser.add_argument('-p', '--push', nargs='+', help='Pushes (with --force) to the given list of remotes from origin/master to their master and production branch, and then fetches them.')
|
|
Theo Chatzimichos |
2b0379 |
parser.add_argument('--checkout', nargs=1, help='Checkout to the specified remote/branch.')
|
|
Theo Chatzimichos |
1f0a34 |
args = parser.parse_args()
|
|
Theo Chatzimichos |
1f0a34 |
|
|
Theo Chatzimichos |
a44a4e |
will_run = False
|
|
Theo Chatzimichos |
a44a4e |
|
|
Theo Chatzimichos |
11d230 |
if args.pull_requests:
|
|
Theo Chatzimichos |
a44a4e |
will_run = True
|
|
Theo Chatzimichos |
11d230 |
check_open_pull_requests()
|
|
Theo Chatzimichos |
968f21 |
|
|
Theo Chatzimichos |
655681 |
if args.remove_symlinks:
|
|
Theo Chatzimichos |
655681 |
will_run = True
|
|
Theo Chatzimichos |
655681 |
remove_symlinks()
|
|
Theo Chatzimichos |
655681 |
|
|
Theo Chatzimichos |
11d230 |
# Every option below requires the --destination argument to be set
|
|
Theo Chatzimichos |
7e56e3 |
if args.clone or args.symlink or args.clone_from or args.clone_branch or args.add_remote or type(args.update) == list or args.push or args.checkout:
|
|
Theo Chatzimichos |
a44a4e |
will_run = True
|
|
Theo Chatzimichos |
a44a4e |
|
|
Theo Chatzimichos |
ab6c8e |
if args.formulas:
|
|
Theo Chatzimichos |
ab6c8e |
unknown_formulas = []
|
|
Theo Chatzimichos |
ab6c8e |
args_formulas = []
|
|
Theo Chatzimichos |
ab6c8e |
FORMULAS = {}
|
|
Theo Chatzimichos |
ab6c8e |
|
|
Theo Chatzimichos |
ab6c8e |
for sublist in args.formulas:
|
|
Theo Chatzimichos |
ab6c8e |
for item in sublist:
|
|
Theo Chatzimichos |
ab6c8e |
args_formulas.append(item)
|
|
Theo Chatzimichos |
ab6c8e |
|
|
Theo Chatzimichos |
ab6c8e |
for formula in args_formulas:
|
|
Theo Chatzimichos |
ab6c8e |
try:
|
|
Theo Chatzimichos |
ab6c8e |
FORMULAS[formula] = FORMULAS_YAML[formula]
|
|
Theo Chatzimichos |
ab6c8e |
except KeyError:
|
|
Theo Chatzimichos |
ab6c8e |
unknown_formulas.append(formula)
|
|
Theo Chatzimichos |
ab6c8e |
if unknown_formulas:
|
|
Theo Chatzimichos |
ab6c8e |
print("ERROR: The following given formulas are not in FORMULAS.yaml: %s\n" % ', '.join(unknown_formulas), file=sys.stderr)
|
|
Theo Chatzimichos |
ab6c8e |
sys.exit(1)
|
|
Theo Chatzimichos |
ab6c8e |
|
|
Theo Chatzimichos |
e2b39c |
if (args.clone_from or args.clone_branch) and not args.clone:
|
|
Theo Chatzimichos |
95d625 |
print('ERROR: Please specify -c / --clone when using --clone-from or --clone-branch', file=sys.stderr)
|
|
Theo Chatzimichos |
e2b39c |
sys.exit(1)
|
|
Theo Chatzimichos |
e2b39c |
|
|
Theo Chatzimichos |
11d230 |
if not args.destination or not os.path.isabs(args.destination[0]):
|
|
Theo Chatzimichos |
95d625 |
print('ERROR: The given destination is not an absolute path', file=sys.stderr)
|
|
Theo Chatzimichos |
8e8456 |
sys.exit(1)
|
|
Theo Chatzimichos |
9af0f5 |
|
|
Theo Chatzimichos |
c0be5f |
if args.add_remote:
|
|
Theo Chatzimichos |
c0be5f |
for remote in args.add_remote:
|
|
Theo Chatzimichos |
c0be5f |
if len(remote) < 2:
|
|
Theo Chatzimichos |
95d625 |
print('ERROR: At least two parameters are required for -r / --add-remote', file=sys.stderr)
|
|
Theo Chatzimichos |
c0be5f |
sys.exit(1)
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
11d230 |
if args.clone:
|
|
Theo Chatzimichos |
e2b39c |
clone_from = None
|
|
Theo Chatzimichos |
e2b39c |
clone_branch = None
|
|
Theo Chatzimichos |
e2b39c |
if args.clone_from:
|
|
Theo Chatzimichos |
e2b39c |
clone_from = args.clone_from[0]
|
|
Theo Chatzimichos |
e2b39c |
if args.clone_branch:
|
|
Theo Chatzimichos |
e2b39c |
clone_branch = args.clone_branch[0]
|
|
Theo Chatzimichos |
042b5b |
clone(clone_from, clone_branch, args.destination[0])
|
|
Theo Chatzimichos |
968f21 |
|
|
Theo Chatzimichos |
11d230 |
if args.symlink:
|
|
Theo Chatzimichos |
11d230 |
create_symlinks(args.destination[0])
|
|
Theo Chatzimichos |
c0be5f |
|
|
Theo Chatzimichos |
c0be5f |
if args.add_remote:
|
|
Theo Chatzimichos |
c0be5f |
add_remote(args.add_remote, args.destination[0])
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
7e56e3 |
if type(args.update) == list:
|
|
Theo Chatzimichos |
25dc6d |
update(args.update, args.destination[0])
|
|
Theo Chatzimichos |
25dc6d |
|
|
Theo Chatzimichos |
25dc6d |
if args.push:
|
|
Theo Chatzimichos |
25dc6d |
push(args.push, args.destination[0])
|
|
Theo Chatzimichos |
a44a4e |
|
|
Theo Chatzimichos |
2b0379 |
if args.checkout:
|
|
Theo Chatzimichos |
2b0379 |
checkout_remote_and_branch(args.checkout[0], args.destination[0])
|
|
Theo Chatzimichos |
2b0379 |
|
|
Theo Chatzimichos |
a44a4e |
if not will_run:
|
|
Theo Chatzimichos |
d99039 |
parser.print_help()
|
|
Theo Chatzimichos |
d99039 |
sys.exit(1)
|