diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e0c7c4..c2d461e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,16 +18,22 @@ test_custom_grains: tags: - docker -test_show_highstate: +test_show_highstate_against_upstream_formulas: stage: test before_script: - - zypper -qn in --no-recommends salt git python3 python3-PyYAML - - rm -rf /srv/{salt,pillar} - - ln -s $PWD/salt /srv/salt - - ln -s $PWD/pillar /srv/pillar - - sed -i -e 's/^production:$/base:/' /srv/{salt,pillar}/top.sls - - bin/get_formulas.py -c /srv/formula -s - script: bin/test_show_highstate.py + - bin/prepare_test_show_highstate_env.sh + - bin/get_formulas.py -c -d /srv/formula -s + script: bin/test_show_highstate.sh + allow_failure: true + tags: + - docker + +test_show_highstate_against_forked_formulas: + stage: test + before_script: + - bin/prepare_test_show_highstate_env.sh + - bin/get_formulas.py -c -d /srv/formula -s -r opensuse + script: bin/test_show_highstate.sh tags: - docker diff --git a/bin/get_formulas.py b/bin/get_formulas.py index f7c314b..5ba402b 100755 --- a/bin/get_formulas.py +++ b/bin/get_formulas.py @@ -4,21 +4,35 @@ import argparse import os +import sys import yaml -def clone(DEST, SYMLINK=False): - def use_git_to_clone_or_pull_repo(): - # pygit2 is not available for python3 in Leap, use plain git instead +def git(cmd, cwd=None, additional_env=None): + # pygit2 is not available for python3 in Leap, use plain git instead + + import subprocess + + env = os.environ.copy() + if additional_env: + env.update(additional_env) - import subprocess + status = subprocess.call(['git'] + cmd, cwd=cwd, env=env) + if status != 0: + sys.exit(status) + +def clone_or_pull(DEST, SYMLINK=False): + def use_git_to_clone_or_pull_repo(): if not os.path.exists(DEST): os.mkdir(DEST) if os.path.isdir(FULL_PATH): - subprocess.Popen(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + git(['pull', '-q'], cwd=FULL_PATH) else: - subprocess.Popen(['git', 'clone', url, FULL_PATH], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + git(['clone', '-q', url, FULL_PATH]) + git(['remote', 'add', 'opensuse', opensuse_fork_url], cwd=FULL_PATH) + # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs + git(['fetch', '-q', 'opensuse'], cwd=FULL_PATH, additional_env={'GIT_SSL_NO_VERIFY': 'true'}) def use_pygit2_to_clone_or_pull_repo(): import pygit2 @@ -33,19 +47,41 @@ def clone(DEST, SYMLINK=False): namespace = data.get('namespace', 'saltstack-formulas') prefix = data.get('prefix', '') url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula) + opensuse_fork_url = 'https://gitlab.infra.opensuse.org/saltstack-formulas/%s-formula' % formula FULL_PATH = '%s/%s-formula' % (DEST, formula) use_git_to_clone_or_pull_repo() if SYMLINK: os.symlink('%s/%s' % (FULL_PATH, formula), '/srv/salt/%s' % formula) +def enable_remote(REMOTE, DEST): + def use_git_to_enable_remote(): + for formula in FORMULAS.keys(): + FULL_PATH = '%s/%s-formula' % (DEST, formula) + git(['checkout', '-qB', 'master', '%s/master' % REMOTE], cwd=FULL_PATH) + + with open('FORMULAS.yaml', 'r') as f: FORMULAS = yaml.load(f) -parser = argparse.ArgumentParser(description='Loads the formulas from FORMULAS.yaml and optionally clones them in a specified destination. Optionally it can also create a symlink from the cloned path to /srv/salt, useful for the CI worker.') -parser.add_argument('-c', '--clone', nargs=1, help='Clones the formulas to a specified destination that is passed as option to this argument.') +parser = argparse.ArgumentParser(description='Loads the formulas from FORMULAS.yaml and optionally clones them in a specified destination. Optionally it can also create a symlink from the cloned path to /srv/salt, useful for the CI worker. The internal gitlab fork will also be added as secondary remote.') +parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination". The gitlab fork will also be added as remote. If the repository is already cloned, then both remotes will be pulled/fetched.') parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.') +parser.add_argument('-r', '--remote', nargs=1, help='Enable the specified remote. Available remotes: origin, opensuse. Default: origin') +requiredArgs = parser.add_argument_group('required arguments') +requiredArgs.add_argument('-d', '--destination', nargs=1, required=True, help='Destination absolute path of the cloned (or to-be-cloned) repositories of the formulas.') args = parser.parse_args() +if not os.path.isabs(args.destination[0]): + parser.print_help() + sys.exit(1) + +if args.remote and args.remote[0] not in ['origin', 'opensuse']: + parser.print_help() + sys.exit(1) + if args.clone: - clone(args.clone[0], args.symlink) + clone_or_pull(args.destination[0], args.symlink) + +if args.remote: + enable_remote(args.remote[0], args.destination[0]) diff --git a/bin/prepare_test_show_highstate_env.sh b/bin/prepare_test_show_highstate_env.sh new file mode 100755 index 0000000..ccb74b9 --- /dev/null +++ b/bin/prepare_test_show_highstate_env.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Prepares the CI runner environment to run the show_highstate tests + +set -e + +if [[ $(whoami) != 'root' ]]; then + if [[ -f /usr/bin/sudo ]]; then + SUDO='/usr/bin/sudo' + else + echo 'Please install sudo first, or run this script as root' + exit 1 + fi +fi + +$SUDO zypper -qn in --no-recommends salt git python3 python3-PyYAML +$SUDO rm -rf /srv/{salt,pillar} +$SUDO ln -s $PWD/salt /srv/salt +$SUDO ln -s $PWD/pillar /srv/pillar +echo 'domain: infra.opensuse.org' | $SUDO tee /etc/salt/grains > /dev/null +sed -i -e 's/^production:$/base:/' /srv/{salt,pillar}/top.sls +ROLES=$(bin/get_roles.py --yaml) +printf "city:\ncountry:\nsalt_cluster: opensuse\nvirt_cluster:\n$ROLES" > pillar/id/${HOSTNAME}.sls diff --git a/bin/test_show_highstate.py b/bin/test_show_highstate.py deleted file mode 100755 index bf70ded..0000000 --- a/bin/test_show_highstate.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Runs state.show_highstate using all localized grains' combinations - -set -e - -RUN_TEST="salt-call --local --retcode-passthrough state.show_highstate" -ROLES=$(bin/get_roles.py --yaml) - -echo 'domain: infra.opensuse.org' > /etc/salt/grains -printf "city:\ncountry:\nsalt_cluster: opensuse\nvirt_cluster:\n$ROLES" > pillar/id/${HOSTNAME}.sls - -sed -i -e 's/\(city:\).*/\1 nuremberg/' -e 's/\(country:\).*/\1 de/' -e 's/\(virt_cluster:\).*/\1 atreju/' pillar/id/${HOSTNAME}.sls -$RUN_TEST > /dev/null -echo "PASSED: country: de" - -sed -i -e 's/\(city:\).*/\1 provo/' -e 's/\(country:\).*/\1 us/' -e 's/\(virt_cluster:\).*/\1 bryce/' pillar/id/${HOSTNAME}.sls -$RUN_TEST > /dev/null -echo "PASSED: country: us" diff --git a/bin/test_show_highstate.sh b/bin/test_show_highstate.sh new file mode 100755 index 0000000..d85d18b --- /dev/null +++ b/bin/test_show_highstate.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Runs state.show_highstate using all localized grains' combinations + +set -e + +RUN_TEST="salt-call --local --retcode-passthrough state.show_highstate" + +sed -i -e 's/\(city:\).*/\1 nuremberg/' -e 's/\(country:\).*/\1 de/' -e 's/\(virt_cluster:\).*/\1 atreju/' pillar/id/${HOSTNAME}.sls +$RUN_TEST > /dev/null +echo "PASSED: country: de" + +sed -i -e 's/\(city:\).*/\1 provo/' -e 's/\(country:\).*/\1 us/' -e 's/\(virt_cluster:\).*/\1 bryce/' pillar/id/${HOSTNAME}.sls +$RUN_TEST > /dev/null +echo "PASSED: country: us"