From 58ee8cba18d2acd0b4a4c750895ecaabdd50dbf1 Mon Sep 17 00:00:00 2001 From: Theo Chatzimichos Date: Nov 13 2017 20:01:08 +0000 Subject: Merge branch 'tampakrap_get_formulas' into 'production' Finalize the improvements on get_formulas for CI runners See merge request infra/salt!90 --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2d461e..fc218b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,9 +30,12 @@ test_show_highstate_against_upstream_formulas: test_show_highstate_against_forked_formulas: stage: test + variables: + # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs + GIT_SSL_NO_VERIFY: 'true' before_script: - bin/prepare_test_show_highstate_env.sh - - bin/get_formulas.py -c -d /srv/formula -s -r opensuse + - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production script: bin/test_show_highstate.sh tags: - docker diff --git a/bin/get_formulas.py b/bin/get_formulas.py index 53e048f..1b3453a 100755 --- a/bin/get_formulas.py +++ b/bin/get_formulas.py @@ -24,31 +24,24 @@ def check_open_pull_requests(): print('%s is %s' % (pull_request, state)) -def git(cmd, cwd=None, additional_env=None): +def git(cmd, cwd=None): # pygit2 is not available for python3 in Leap, use plain git instead import subprocess - env = os.environ.copy() - if additional_env: - env.update(additional_env) - - status = subprocess.call(['git'] + cmd, cwd=cwd, env=env) + status = subprocess.call(['git'] + cmd, cwd=cwd) if status != 0: sys.exit(status) -def clone_or_pull(DEST): +def clone_or_pull(CLONE_FROM, CLONE_BRANCH, DEST): def use_git_to_clone_or_pull_repo(): if not os.path.exists(DEST): os.mkdir(DEST) if os.path.isdir(FULL_PATH): git(['pull', '-q'], cwd=FULL_PATH) else: - git(['clone', '-q', url, FULL_PATH]) - git(['remote', 'add', 'opensuse', opensuse_fork_url], cwd=FULL_PATH) - # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs - git(['fetch', '-q', 'opensuse'], cwd=FULL_PATH, additional_env={'GIT_SSL_NO_VERIFY': 'true'}) + git(['clone', '-q', url, FULL_PATH] + branch_opts) def use_pygit2_to_clone_or_pull_repo(): import pygit2 @@ -59,13 +52,21 @@ def clone_or_pull(DEST): else: pygit2.clone_repository(url, FULL_PATH, bare=False) - for formula, data in FORMULAS.items(): - namespace = data.get('namespace', 'saltstack-formulas') - prefix = data.get('prefix', '') - url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula) - opensuse_fork_url = 'https://gitlab.infra.opensuse.org/saltstack-formulas/%s-formula' % formula - FULL_PATH = '%s/%s-formula' % (DEST, formula) - use_git_to_clone_or_pull_repo() + branch_opts = [] + if CLONE_BRANCH: + branch_opts = ['-b', CLONE_BRANCH, '--single-branch'] + if CLONE_FROM: + for formula in FORMULAS.keys(): + url = '%s/%s-formula' % (CLONE_FROM, formula) + FULL_PATH = '%s/%s-formula' % (DEST, formula) + use_git_to_clone_or_pull_repo() + else: + for formula, data in FORMULAS.items(): + namespace = data.get('namespace', 'saltstack-formulas') + prefix = data.get('prefix', '') + url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula) + FULL_PATH = '%s/%s-formula' % (DEST, formula) + use_git_to_clone_or_pull_repo() def create_symlinks(DEST): @@ -73,41 +74,42 @@ def create_symlinks(DEST): os.symlink('%s/%s-formula/%s' % (DEST, formula, formula), '/srv/salt/%s' % formula) -def enable_remote(REMOTE, DEST): - def use_git_to_enable_remote(): - for formula in FORMULAS.keys(): - FULL_PATH = '%s/%s-formula' % (DEST, formula) - git(['checkout', '-qB', 'master', '%s/master' % REMOTE], cwd=FULL_PATH) - - with open('FORMULAS.yaml', 'r') as f: FORMULAS = yaml.load(f) -parser = argparse.ArgumentParser(description='Loads the formulas from FORMULAS.yaml and optionally clones them in a specified destination. Optionally it can also create a symlink from the cloned path to /srv/salt, useful for the CI worker. The internal gitlab fork will also be added as secondary remote.') +parser = argparse.ArgumentParser(description='Loads the formulas from FORMULAS.yaml and optionally clones them in a specified destination. Optionally it can also create a symlink from the cloned path to /srv/salt, useful for the CI worker.') parser.add_argument('-p', '--pull-requests', action='store_true', help='Prints the status of the Pull Requests that are defined in FORMULAS.yaml under "pending".') parser.add_argument('-d', '--destination', nargs=1, help='Destination absolute path of the cloned (or to-be-cloned) repositories of the formulas.') -parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination". The gitlab fork will also be added as remote. If the repository is already cloned, then both remotes will be pulled/fetched.') +parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination". If the repository is already cloned, then it will be pulled/fetched.') +parser.add_argument('--clone-from', nargs=1, help='Specify the git provider to clone from together with the namespace.') +parser.add_argument('--clone-branch', nargs=1, help='Specify the branch to clone.') parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.') -parser.add_argument('-r', '--remote', nargs=1, help='Enable the specified remote. Available remotes: origin, opensuse. Default: origin') args = parser.parse_args() if args.pull_requests: check_open_pull_requests() # Every option below requires the --destination argument to be set -if args.clone or args.symlink or args.remote: +if args.clone or args.symlink or args.clone_from or args.clone_branch: + if (args.clone_from or args.clone_branch) and not args.clone: + parser.print_help() + sys.exit(1) + if not args.destination or not os.path.isabs(args.destination[0]): parser.print_help() sys.exit(1) if args.clone: - clone_or_pull(args.destination[0]) + clone_from = None + clone_branch = None + if args.clone_from: + clone_from = args.clone_from[0] + if args.clone_branch: + clone_branch = args.clone_branch[0] + clone_or_pull(clone_from, clone_branch, args.destination[0]) if args.symlink: create_symlinks(args.destination[0]) - - if args.remote: - if args.remote[0] not in ['origin', 'opensuse']: - parser.print_help() - sys.exit(1) - enable_remote(args.remote[0], args.destination[0]) +else: + parser.print_help() + sys.exit(1)