From deba4ad2691b7bee53e7f0a3e2a3c45cda1dd1d0 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Feb 16 2024 20:52:10 +0000 Subject: Drop get_formulas.py No longer useful since Salt no longer allows states to be installed outside the file root - and the /srv/formula directory populated by the script is not mountable as a file root itself given the states residing in subdirectories under the respective formula repository directories. The installation of Git based formulas in the test environment is taken care of by clone_formulas.sh now, which populates /srv/formulas with a checkout of salt-formulas-git.git, which in return was equipped with top level state directory symlinks. https://gitlab.infra.opensuse.org/infra/salt/-/issues/20 https://bugzilla.opensuse.org/show_bug.cgi?id=1219985 Signed-off-by: Georg Pfuetzenreuter --- diff --git a/.gitlab-ci.templates/test_highstate.jinja b/.gitlab-ci.templates/test_highstate.jinja index 15434f2..1ad3781 100644 --- a/.gitlab-ci.templates/test_highstate.jinja +++ b/.gitlab-ci.templates/test_highstate.jinja @@ -3,7 +3,6 @@ stage: test before_script: - bin/prepare_test_env.sh -g -o Leap,15,5 -s -n - - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' tags: - docker artifacts: diff --git a/.gitlab-ci.templates/test_nginx.jinja b/.gitlab-ci.templates/test_nginx.jinja index 86498fc..ccbf2a8 100644 --- a/.gitlab-ci.templates/test_nginx.jinja +++ b/.gitlab-ci.templates/test_nginx.jinja @@ -3,7 +3,6 @@ stage: test before_script: - bin/prepare_test_env.sh -g -s - - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' -f nginx script: bin/test_nginx.sh tags: - docker diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fec06e..290d1f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,6 @@ show_highstate: - .test_artifacts before_script: - bin/prepare_test_env.sh -g -o Leap,15,4 -s - - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' script: bin/test_show_highstate.sh test_haproxy: @@ -50,7 +49,6 @@ test_haproxy: before_script: - bin/prepare_test_env.sh -g -s -n - bin/replace_secrets.sh - - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' -f haproxy script: bin/test_haproxy.sh # changes: # - bin/test_haproxy.sh diff --git a/bin/get_formulas.py b/bin/get_formulas.py deleted file mode 100755 index 5305865..0000000 --- a/bin/get_formulas.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python3 - -# For description and usage, see the argparse options at the end of the file - -import argparse -import os -import sys -from copy import copy - -import pygit2 -import yaml -from pygit2.errors import GitError - - -def git(cmd, cwd=None): - # TODO migrate to pygit2 - - import subprocess - - status = subprocess.call(['git'] + cmd, cwd=cwd) - if status != 0: - sys.exit(status) - - -def clone(CLONE_FROM, DEST): - def clone_repo(): - FULL_PATH = f'{DEST}/{formula}-formula' - if os.path.isdir(FULL_PATH): - return - - pygit2.clone_repository(url, FULL_PATH, bare=False) - - if CLONE_FROM: - for formula in FORMULAS.keys(): - url = f'{CLONE_FROM}/{formula}-formula' - clone_repo() - else: - for formula, data in FORMULAS.items(): - namespace = data.get('namespace', 'saltstack-formulas') - prefix = data.get('prefix', '') - url = f'https://github.com/{namespace}/{prefix}{formula}-formula' - clone_repo() - - -def create_symlinks(DEST): - for formula in FORMULAS.keys(): - FULL_PATH = '/srv/salt/%s' % formula - if not os.path.islink(FULL_PATH): - os.symlink(f'{DEST}/{formula}-formula/{formula}', FULL_PATH) - - -def remove_symlinks(): - for formula in FORMULAS.keys(): - FULL_PATH = '/srv/salt/%s' % formula - if os.path.islink(FULL_PATH): - os.unlink(FULL_PATH) - - -def fetch_remote(remote, formula): - remotecallbacks = None - if not remote.url.startswith(('http://', 'https://', 'git://', 'ssh://', 'git+ssh://')): - username = remote.url.split('@')[0] - credentials = pygit2.KeypairFromAgent(username) - remotecallbacks = pygit2.RemoteCallbacks(credentials=credentials) - try: - remote.fetch(callbacks=remotecallbacks) - except GitError: - print(f'{formula}-formula: Failed to fetch remote {remote.name}') - - -def add_remote(REMOTES, DEST): - for remote in REMOTES: - namespace = None - if len(remote) == 4: - namespace = remote.pop() - - url = 'https://github.com' - if len(remote) == 3: - url = remote.pop() - - prefix = '' - use_prefix = False - if not remote[1].startswith('no'): - use_prefix = True - - name = remote[0] - - for formula, data in FORMULAS.items(): - if not namespace: - namespace = data.get('namespace', 'saltstack-formulas') - if use_prefix: - prefix = data.get('prefix', '') - if not url.endswith(':'): - url += '/' - full_url = f'{url}{namespace}/{prefix}{formula}-formula' - FULL_PATH = f'{DEST}/{formula}-formula' - repo = pygit2.Repository(FULL_PATH) - try: - repo.create_remote(name, full_url) - except ValueError: # remote already exists - continue - fetch_remote(repo.remotes[name], formula) - - -def update(REMOTES, DEST): - for formula in FORMULAS.keys(): - FULL_PATH = f'{DEST}/{formula}-formula' - repo = pygit2.Repository(FULL_PATH) - git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH) - git(['pull', '-q'], cwd=FULL_PATH) - if REMOTES: - for remote in REMOTES: - fetch_remote(repo.remotes[remote], formula) - - -def push(REMOTES, DEST): - for formula in FORMULAS.keys(): - FULL_PATH = f'{DEST}/{formula}-formula' - repo = pygit2.Repository(FULL_PATH) - git(['checkout', '-qB', 'master', 'origin/master'], cwd=FULL_PATH) - for remote in REMOTES: - git(['push', '-qf', remote, 'master'], cwd=FULL_PATH) - git(['push', '-qf', remote, 'master:production'], cwd=FULL_PATH) - fetch_remote(repo.remotes[remote], formula) - - -def checkout_remote_and_branch(REMOTE_BRANCH, DEST): - for formula in FORMULAS.keys(): - FULL_PATH = f'{DEST}/{formula}-formula' - branch = REMOTE_BRANCH.split('/')[1] - git(['checkout', '-qB', branch, REMOTE_BRANCH], cwd=FULL_PATH) - - -with open('pillar/FORMULAS.yaml', 'r') as f: - FORMULAS_YAML = yaml.safe_load(f) - -FORMULAS = copy(FORMULAS_YAML)['git'] - -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description='Loads the formulas from FORMULAS.yaml and performs one or more of the operations specified at the arguments.') -parser.add_argument('-d', '--destination', nargs=1, help='Destination absolute path of the cloned (or to-be-cloned) repositories of the formulas.') -parser.add_argument('-f', '--formulas', action='append', nargs='+', help='Specify specific formulas to operate on, instead of working with all the specified FORMULAS.yaml formulas.') -parser.add_argument('-c', '--clone', action='store_true', help='Clone the formulas to the destination specified with "--destination".') -parser.add_argument('--clone-from', nargs=1, help='Specify the git provider to clone from together with the namespace.') -parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.') -parser.add_argument('--remove-symlinks', action='store_true', help='Removes all symlinks that were created in /srv/salt.') -parser.add_argument('-r', '--add-remote', action='append', nargs='+', help='''Add the specified remotes on the local repositories. It can be passed multiple times. -Usage: REMOTE_NAME USE_PREFIXES [GIT_PROVIDER_URL] [NAMESPACE]. - - REMOTE is string - - USE_PREFIXES should be a string starting with "no" (for no prefix usage), or whatever else string (for prefix usage) - - GIT_URL (optional) can be in the form "https://gitlab.example.com" or "git@gitlab.example.com:" (make sure you have the trailing colon). If no git provider URL is given, https://github.com will be used. - - NAMESPACE (optional) is string. If no namespace is given, the one defined in FORMULAS.yaml will be used. -Examples: - -r forks_ro prefixes - -r forks_rw prefixes git@github.com: - -r mycompany no_prefixes https://gitlab.mycompany.com saltstack-formulas - -r mycompany_forks no_prefixes git@gitlab.mycompany.com: saltstack-formulas''') -parser.add_argument('-u', '--update', nargs='*', help='Switch to origin/master and git pull. Optionally it can accept a list of remotes as arguments, that will be fetched.') -parser.add_argument('-p', '--push', nargs='+', help='Pushes (with --force) to the given list of remotes from origin/master to their master and production branch, and then fetches them.') -parser.add_argument('--checkout', nargs=1, help='Checkout to the specified remote/branch.') -args = parser.parse_args() - -will_run = False - -if args.remove_symlinks: - will_run = True - remove_symlinks() - -# Every option below requires the --destination argument to be set -if args.clone or args.symlink or args.clone_from or args.add_remote or isinstance(args.update, list) or args.push or args.checkout: - will_run = True - - if args.formulas: - unknown_formulas = [] - args_formulas = [] - FORMULAS = {} - - for sublist in args.formulas: - for item in sublist: - args_formulas.append(item) - - for formula in args_formulas: - try: - FORMULAS[formula] = FORMULAS_YAML['git'][formula] - except KeyError: - unknown_formulas.append(formula) - if unknown_formulas: - print("ERROR: The following given formulas are not in FORMULAS.yaml: %s\n" % ', '.join(unknown_formulas), file=sys.stderr) - sys.exit(1) - - if args.clone_from and not args.clone: - print('ERROR: Please specify -c / --clone when using --clone-from', file=sys.stderr) - sys.exit(1) - - if not args.destination or not os.path.isabs(args.destination[0]): - print('ERROR: The given destination is not an absolute path', file=sys.stderr) - sys.exit(1) - - if args.add_remote: - for remote in args.add_remote: - if len(remote) < 2: - print('ERROR: At least two parameters are required for -r / --add-remote', file=sys.stderr) - sys.exit(1) - - if args.clone: - clone_from = None - if args.clone_from: - clone_from = args.clone_from[0] - clone(clone_from, args.destination[0]) - - if args.symlink: - create_symlinks(args.destination[0]) - - if args.add_remote: - add_remote(args.add_remote, args.destination[0]) - - if isinstance(args.update, list): - update(args.update, args.destination[0]) - - if args.push: - push(args.push, args.destination[0]) - - if args.checkout: - checkout_remote_and_branch(args.checkout[0], args.destination[0]) - -if not will_run: - parser.print_help() - sys.exit(1) diff --git a/bin/run_tests_locally.sh b/bin/run_tests_locally.sh index d3cf21e..c063d84 100755 --- a/bin/run_tests_locally.sh +++ b/bin/run_tests_locally.sh @@ -138,9 +138,6 @@ $SSH sh <<-EOS || echo 'Test suite returned with errors.' sudo bin/prepare_test_env.sh -g -s || pfail sudo sed -i 's/download-prg.infra.opensuse.org/download.opensuse.org/' /etc/zypp/repos.d/* - pprint 'Preparing formulas' - bin/get_formulas.py -c -d /srv/salt-formulas -s --clone-from https://gitlab.infra.opensuse.org/saltstack-formulas || pfail - pprint Testing: validate bin/test_validate.sh || pfail diff --git a/t/lib/Dockerfile.systemd b/t/lib/Dockerfile.systemd index ea3e259..0559669 100644 --- a/t/lib/Dockerfile.systemd +++ b/t/lib/Dockerfile.systemd @@ -10,7 +10,7 @@ RUN test ! -f /var/log/zypper.log || mv /var/log/zypper.log /var/log/zypper.log. # these are needed to run test RUN zypper ar https://download.opensuse.org/repositories/openSUSE:/infrastructure/\$releasever/ infra && \ zypper --gpg-auto-import-keys ref -RUN ( zypper -vvvn ref || zypper -vvvn ref || zypper -vvvn ref ) && zypper -vvvn install systemd salt-minion curl sudo iputils python3-pygit2 ca-certificates-suse ca-certificates-freeipa-opensuse zypper-formula # needed for get_formulas.py +RUN ( zypper -vvvn ref || zypper -vvvn ref || zypper -vvvn ref ) && zypper -vvvn install systemd salt-minion curl sudo iputils ca-certificates-suse ca-certificates-freeipa-opensuse zypper-formula ##DUMMY @@ -21,10 +21,11 @@ RUN mkdir -p /srv/salt/ && \ ADD bin /test/bin ADD pillar /test/pillar WORKDIR /test -RUN bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' || bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' +RUN bin/clone_formulas.sh RUN ( echo $'file_roots:\n\ base:\n\ - /srv/salt\n\ + - /srv/formulas\n\ - /usr/share/salt-formulas/states' \ ) >> /etc/salt/minion.d/roots.conf