Theo Chatzimichos 1f0a34
#!/usr/bin/python3
Theo Chatzimichos 1f0a34
Theo Chatzimichos 1f0a34
# For description and usage, see the argparse options at the end of the file
Theo Chatzimichos 1f0a34
Theo Chatzimichos ab6c8e
from copy import copy
Theo Chatzimichos 042b5b
from pygit2.errors import GitError
Theo Chatzimichos 1f0a34
import argparse
Theo Chatzimichos 1f0a34
import os
Theo Chatzimichos 042b5b
import pygit2
Theo Chatzimichos 4af2b2
import sys
Theo Chatzimichos 1f0a34
import yaml
Theo Chatzimichos 1f0a34
Theo Chatzimichos 1f0a34
Theo Chatzimichos 11d230
def check_open_pull_requests():
Theo Chatzimichos 11d230
    from github import Github
Theo Chatzimichos 11d230
Theo Chatzimichos 11d230
    g = Github()
Theo Chatzimichos 11d230
    for formula, data in FORMULAS.items():
Theo Chatzimichos 11d230
        open_pull_requests = data.get('pending', [])
Theo Chatzimichos 11d230
        if open_pull_requests:
Theo Chatzimichos 11d230
            namespace = data.get('original_namespace', 'saltstack-formulas')
Theo Chatzimichos 11d230
            prefix = data.get('prefix', '')
Theo Chatzimichos 11d230
            org = g.get_organization(namespace)
Theo Chatzimichos 11d230
            for pull_request in open_pull_requests:
Theo Chatzimichos 11d230
                pr = int(pull_request.split('/')[-1])
Theo Chatzimichos 11d230
                state = org.get_repo('%s%s-formula' % (prefix, formula)).get_pull(pr).state
Theo Chatzimichos 11d230
                print('%s is %s' % (pull_request, state))
Theo Chatzimichos 11d230
Theo Chatzimichos 11d230
Theo Chatzimichos 62825a
def git(cmd, cwd=None):
Theo Chatzimichos 042b5b
    # TODO migrate to pygit2
Theo Chatzimichos 7e97fe
Theo Chatzimichos 7e97fe
    import subprocess
Theo Chatzimichos 7e97fe
Theo Chatzimichos 62825a
    status = subprocess.call(['git'] + cmd, cwd=cwd)
Theo Chatzimichos 7e97fe
    if status != 0:
Theo Chatzimichos 7e97fe
        sys.exit(status)
Theo Chatzimichos 1f0a34
Theo Chatzimichos 1f0a34
Theo Chatzimichos 042b5b
def clone(CLONE_FROM, CLONE_BRANCH, DEST):
Theo Chatzimichos ce9231
    def clone_repo():
Theo Chatzimichos ce9231
        FULL_PATH = '%s/%s-formula' % (DEST, formula)
Theo Chatzimichos ce9231
        if os.path.isdir(FULL_PATH):
Theo Chatzimichos ce9231
            return
Theo Chatzimichos 1f0a34
Theo Chatzimichos 042b5b
        pygit2.clone_repository(url, FULL_PATH, bare=False)
Theo Chatzimichos 1f0a34
Theo Chatzimichos e2b39c
    branch_opts = []
Theo Chatzimichos e2b39c
    if CLONE_BRANCH:
Theo Chatzimichos e2b39c
        branch_opts = ['-b', CLONE_BRANCH, '--single-branch']
Theo Chatzimichos e2b39c
    if CLONE_FROM:
Theo Chatzimichos e2b39c
        for formula in FORMULAS.keys():
Theo Chatzimichos e2b39c
            url = '%s/%s-formula' % (CLONE_FROM, formula)
Theo Chatzimichos ce9231
            clone_repo()
Theo Chatzimichos e2b39c
    else:
Theo Chatzimichos e2b39c
        for formula, data in FORMULAS.items():
Theo Chatzimichos e2b39c
            namespace = data.get('namespace', 'saltstack-formulas')
Theo Chatzimichos e2b39c
            prefix = data.get('prefix', '')
Theo Chatzimichos e2b39c
            url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula)
Theo Chatzimichos ce9231
            clone_repo()
Theo Chatzimichos 9af0f5
Theo Chatzimichos 9af0f5
Theo Chatzimichos 9af0f5
def create_symlinks(DEST):
Theo Chatzimichos 9af0f5
    for formula in FORMULAS.keys():
Theo Chatzimichos 16b98e
        FULL_PATH = '/srv/salt/%s' % formula
Theo Chatzimichos 16b98e
        if not os.path.islink(FULL_PATH):
Theo Chatzimichos 16b98e
            os.symlink('%s/%s-formula/%s' % (DEST, formula, formula), FULL_PATH)
Theo Chatzimichos 1f0a34
Theo Chatzimichos 1f0a34
Theo Chatzimichos 655681
def remove_symlinks():
Theo Chatzimichos 655681
    for formula in FORMULAS.keys():
Theo Chatzimichos 655681
        FULL_PATH = '/srv/salt/%s' % formula
Theo Chatzimichos 655681
        if os.path.islink(FULL_PATH):
Theo Chatzimichos 655681
            os.unlink(FULL_PATH)
Theo Chatzimichos 655681
Theo Chatzimichos 655681
Theo Chatzimichos 25dc6d
def fetch_remote(remote, formula):
Theo Chatzimichos a900e2
    remotecallbacks = None
Theo Chatzimichos a900e2
    if not remote.url.startswith(('http://', 'https://', 'git://', 'ssh://', 'git+ssh://')):
Theo Chatzimichos a900e2
        username = remote.url.split('@')[0]
Theo Chatzimichos a900e2
        credentials = pygit2.KeypairFromAgent(username)
Theo Chatzimichos a900e2
        remotecallbacks = pygit2.RemoteCallbacks(credentials=credentials)
Theo Chatzimichos 25dc6d
    try:
Theo Chatzimichos a900e2
        remote.fetch(callbacks=remotecallbacks)
Theo Chatzimichos 25dc6d
    except GitError:
Theo Chatzimichos 25dc6d
        print('%s-formula: Failed to fetch remote %s' % (formula, remote.name))
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
def add_remote(REMOTES, DEST):
Theo Chatzimichos c0be5f
    for remote in REMOTES:
Theo Chatzimichos c0be5f
        namespace = None
Theo Chatzimichos c0be5f
        if len(remote) == 4:
Theo Chatzimichos c0be5f
            namespace = remote.pop()
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
        url = 'https://github.com'
Theo Chatzimichos c0be5f
        if len(remote) == 3:
Theo Chatzimichos c0be5f
            url = remote.pop()
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
        prefix = ''
Theo Chatzimichos c0be5f
        use_prefix = False
Theo Chatzimichos c0be5f
        if not remote[1].startswith('no'):
Theo Chatzimichos c0be5f
            use_prefix = True
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
        name = remote[0]
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
        for formula, data in FORMULAS.items():
Theo Chatzimichos c0be5f
            if not namespace:
Theo Chatzimichos c0be5f
                namespace = data.get('namespace', 'saltstack-formulas')
Theo Chatzimichos c0be5f
            if use_prefix:
Theo Chatzimichos c0be5f
                prefix = data.get('prefix', '')
Theo Chatzimichos c0be5f
            if not url.endswith(':'):
Theo Chatzimichos c0be5f
                url += '/'
Theo Chatzimichos c0be5f
            full_url = '%s%s/%s%s-formula' % (url, namespace, prefix, formula)
Theo Chatzimichos c0be5f
            FULL_PATH = '%s/%s-formula' % (DEST, formula)
Theo Chatzimichos c0be5f
            repo = pygit2.Repository(FULL_PATH)
Theo Chatzimichos c0be5f
            try:
Theo Chatzimichos c0be5f
                repo.create_remote(name, full_url)
Theo Chatzimichos ec31ef
            except ValueError:  # remote already exists
Theo Chatzimichos ec31ef
                continue
Theo Chatzimichos 25dc6d
            fetch_remote(repo.remotes[name], formula)
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
def update(REMOTES, DEST):
Theo Chatzimichos 25dc6d
    for formula in FORMULAS.keys():
Theo Chatzimichos 25dc6d
        FULL_PATH = '%s/%s-formula' % (DEST, formula)
Theo Chatzimichos 25dc6d
        repo = pygit2.Repository(FULL_PATH)
Theo Chatzimichos 25dc6d
        git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH)
Theo Chatzimichos 25dc6d
        git(['pull', '-q'], cwd=FULL_PATH)
Theo Chatzimichos 25dc6d
        if REMOTES:
Theo Chatzimichos 25dc6d
            for remote in REMOTES:
Theo Chatzimichos 25dc6d
                fetch_remote(repo.remotes[remote], formula)
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
def push(REMOTES, DEST):
Theo Chatzimichos 25dc6d
    for formula in FORMULAS.keys():
Theo Chatzimichos 25dc6d
        FULL_PATH = '%s/%s-formula' % (DEST, formula)
Theo Chatzimichos 25dc6d
        repo = pygit2.Repository(FULL_PATH)
Theo Chatzimichos 25dc6d
        git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH)
Theo Chatzimichos 25dc6d
        for remote in REMOTES:
Theo Chatzimichos 25dc6d
            git(['push', '-qf', remote, 'master'], cwd=FULL_PATH)
Theo Chatzimichos 25dc6d
            git(['push', '-qf', remote, 'master:production'], cwd=FULL_PATH)
Theo Chatzimichos 25dc6d
            fetch_remote(repo.remotes[remote], formula)
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
Theo Chatzimichos 2b0379
def checkout_remote_and_branch(REMOTE_BRANCH, DEST):
Theo Chatzimichos 2b0379
    for formula in FORMULAS.keys():
Theo Chatzimichos 2b0379
        FULL_PATH = '%s/%s-formula' % (DEST, formula)
Theo Chatzimichos 2b0379
        branch = REMOTE_BRANCH.split('/')[1]
Theo Chatzimichos 2b0379
        git(['checkout', '-qB', branch, REMOTE_BRANCH], cwd=FULL_PATH)
Theo Chatzimichos 2b0379
Theo Chatzimichos 2b0379
Theo Chatzimichos 44c07a
with open('pillar/FORMULAS.yaml', 'r') as f:
Karol Babioch 3016f1
    FORMULAS_YAML = yaml.safe_load(f)
Theo Chatzimichos ab6c8e
Theo Chatzimichos ab6c8e
FORMULAS = copy(FORMULAS_YAML)
Theo Chatzimichos 1f0a34
Theo Chatzimichos c0be5f
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description='Loads the formulas from FORMULAS.yaml and performs one or more of the operations specified at the arguments.')
Theo Chatzimichos 25dc6d
parser.add_argument('-q', '--pull-requests', action='store_true', help='Prints the status of the Pull Requests that are defined in FORMULAS.yaml under "pending".')
Theo Chatzimichos 8e8456
parser.add_argument('-d', '--destination', nargs=1, help='Destination absolute path of the cloned (or to-be-cloned) repositories of the formulas.')
Theo Chatzimichos ab6c8e
parser.add_argument('-f', '--formulas', action='append', nargs='+', help='Specify specific formulas to operate on, instead of working with all the specified FORMULAS.yaml formulas.')
Theo Chatzimichos cd2f66
parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination".')
Theo Chatzimichos e2b39c
parser.add_argument('--clone-from', nargs=1, help='Specify the git provider to clone from together with the namespace.')
Theo Chatzimichos e2b39c
parser.add_argument('--clone-branch', nargs=1, help='Specify the branch to clone.')
Theo Chatzimichos 1f0a34
parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.')
Theo Chatzimichos 655681
parser.add_argument('--remove-symlinks', action='store_true', help='Removes all symlinks that were created in /srv/salt.')
Theo Chatzimichos c0be5f
parser.add_argument('-r', '--add-remote', action='append', nargs='+', help='''Add the specified remotes on the local repositories. It can be passed multiple times.
Theo Chatzimichos c0be5f
Usage: REMOTE_NAME USE_PREFIXES [GIT_PROVIDER_URL] [NAMESPACE].
Theo Chatzimichos c0be5f
       - REMOTE is string
Theo Chatzimichos c0be5f
       - USE_PREFIXES should be a string starting with "no" (for no prefix usage), or whatever else string (for prefix usage)
Theo Chatzimichos c0be5f
       - GIT_URL (optional) can be in the form "https://gitlab.example.com" or "git@gitlab.example.com:" (make sure you have the trailing colon). If no git provider URL is given, https://github.com will be used.
Theo Chatzimichos c0be5f
       - NAMESPACE (optional) is string. If no namespace is given, the one defined in FORMULAS.yaml will be used.
Theo Chatzimichos c0be5f
Examples:
Theo Chatzimichos c0be5f
         -r forks_ro prefixes
Theo Chatzimichos c0be5f
         -r forks_rw prefixes git@github.com:
Theo Chatzimichos c0be5f
         -r mycompany no_prefixes https://gitlab.mycompany.com saltstack-formulas
Theo Chatzimichos c0be5f
         -r mycompany_forks no_prefixes git@gitlab.mycompany.com: saltstack-formulas''')
Theo Chatzimichos 25dc6d
parser.add_argument('-u', '--update', nargs='*', help='Switch to origin/master and git pull. Optionally it can accept a list of remotes as arguments, that will be fetched.')
Theo Chatzimichos 25dc6d
parser.add_argument('-p', '--push', nargs='+', help='Pushes (with --force) to the given list of remotes from origin/master to their master and production branch, and then fetches them.')
Theo Chatzimichos 2b0379
parser.add_argument('--checkout', nargs=1, help='Checkout to the specified remote/branch.')
Theo Chatzimichos 1f0a34
args = parser.parse_args()
Theo Chatzimichos 1f0a34
Theo Chatzimichos a44a4e
will_run = False
Theo Chatzimichos a44a4e
Theo Chatzimichos 11d230
if args.pull_requests:
Theo Chatzimichos a44a4e
    will_run = True
Theo Chatzimichos 11d230
    check_open_pull_requests()
Theo Chatzimichos 968f21
Theo Chatzimichos 655681
if args.remove_symlinks:
Theo Chatzimichos 655681
    will_run = True
Theo Chatzimichos 655681
    remove_symlinks()
Theo Chatzimichos 655681
Theo Chatzimichos 11d230
# Every option below requires the --destination argument to be set
Theo Chatzimichos 7e56e3
if args.clone or args.symlink or args.clone_from or args.clone_branch or args.add_remote or type(args.update) == list or args.push or args.checkout:
Theo Chatzimichos a44a4e
    will_run = True
Theo Chatzimichos a44a4e
Theo Chatzimichos ab6c8e
    if args.formulas:
Theo Chatzimichos ab6c8e
        unknown_formulas = []
Theo Chatzimichos ab6c8e
        args_formulas = []
Theo Chatzimichos ab6c8e
        FORMULAS = {}
Theo Chatzimichos ab6c8e
Theo Chatzimichos ab6c8e
        for sublist in args.formulas:
Theo Chatzimichos ab6c8e
            for item in sublist:
Theo Chatzimichos ab6c8e
                args_formulas.append(item)
Theo Chatzimichos ab6c8e
Theo Chatzimichos ab6c8e
        for formula in args_formulas:
Theo Chatzimichos ab6c8e
            try:
Theo Chatzimichos ab6c8e
                FORMULAS[formula] = FORMULAS_YAML[formula]
Theo Chatzimichos ab6c8e
            except KeyError:
Theo Chatzimichos ab6c8e
                unknown_formulas.append(formula)
Theo Chatzimichos ab6c8e
        if unknown_formulas:
Theo Chatzimichos ab6c8e
            print("ERROR: The following given formulas are not in FORMULAS.yaml: %s\n" % ', '.join(unknown_formulas), file=sys.stderr)
Theo Chatzimichos ab6c8e
            sys.exit(1)
Theo Chatzimichos ab6c8e
Theo Chatzimichos e2b39c
    if (args.clone_from or args.clone_branch) and not args.clone:
Theo Chatzimichos 95d625
        print('ERROR: Please specify -c / --clone when using --clone-from or --clone-branch', file=sys.stderr)
Theo Chatzimichos e2b39c
        sys.exit(1)
Theo Chatzimichos e2b39c
Theo Chatzimichos 11d230
    if not args.destination or not os.path.isabs(args.destination[0]):
Theo Chatzimichos 95d625
        print('ERROR: The given destination is not an absolute path', file=sys.stderr)
Theo Chatzimichos 8e8456
        sys.exit(1)
Theo Chatzimichos 9af0f5
Theo Chatzimichos c0be5f
    if args.add_remote:
Theo Chatzimichos c0be5f
        for remote in args.add_remote:
Theo Chatzimichos c0be5f
            if len(remote) < 2:
Theo Chatzimichos 95d625
                print('ERROR: At least two parameters are required for -r / --add-remote', file=sys.stderr)
Theo Chatzimichos c0be5f
                sys.exit(1)
Theo Chatzimichos c0be5f
Theo Chatzimichos 11d230
    if args.clone:
Theo Chatzimichos e2b39c
        clone_from = None
Theo Chatzimichos e2b39c
        clone_branch = None
Theo Chatzimichos e2b39c
        if args.clone_from:
Theo Chatzimichos e2b39c
            clone_from = args.clone_from[0]
Theo Chatzimichos e2b39c
        if args.clone_branch:
Theo Chatzimichos e2b39c
            clone_branch = args.clone_branch[0]
Theo Chatzimichos 042b5b
        clone(clone_from, clone_branch, args.destination[0])
Theo Chatzimichos 968f21
Theo Chatzimichos 11d230
    if args.symlink:
Theo Chatzimichos 11d230
        create_symlinks(args.destination[0])
Theo Chatzimichos c0be5f
Theo Chatzimichos c0be5f
    if args.add_remote:
Theo Chatzimichos c0be5f
        add_remote(args.add_remote, args.destination[0])
Theo Chatzimichos 25dc6d
Theo Chatzimichos 7e56e3
    if type(args.update) == list:
Theo Chatzimichos 25dc6d
        update(args.update, args.destination[0])
Theo Chatzimichos 25dc6d
Theo Chatzimichos 25dc6d
    if args.push:
Theo Chatzimichos 25dc6d
        push(args.push, args.destination[0])
Theo Chatzimichos a44a4e
Theo Chatzimichos 2b0379
    if args.checkout:
Theo Chatzimichos 2b0379
        checkout_remote_and_branch(args.checkout[0], args.destination[0])
Theo Chatzimichos 2b0379
Theo Chatzimichos a44a4e
if not will_run:
Theo Chatzimichos d99039
    parser.print_help()
Theo Chatzimichos d99039
    sys.exit(1)