diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b78cfb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin/__pycache__ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 543e51b..44534b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - test - deploy -test_roles_job: +test_roles: stage: test before_script: - zypper -qn in --no-recommends python3 python3-PyYAML @@ -10,6 +10,19 @@ test_roles_job: tags: - docker +test_show_highstate: + stage: test + before_script: + - zypper -qn in --no-recommends salt git python3 python3-PyYAML + - rm -rf /srv/{salt,pillar} + - ln -s $PWD/salt /srv/salt + - ln -s $PWD/pillar /srv/pillar + - sed -i -e 's/^production:$/base:/' /srv/{salt,pillar}/top.sls + - bin/get_formulas.py -c /srv/formula -s + script: bin/test_show_highstate.py + tags: + - docker + deploy_job: stage: deploy script: salt-call event.fire_master update salt/fileserver/gitfs/update diff --git a/FORMULAS.yaml b/FORMULAS.yaml new file mode 100644 index 0000000..dbc3874 --- /dev/null +++ b/FORMULAS.yaml @@ -0,0 +1,31 @@ +--- +dhcpd: {} +elasticsearch: + namespace: 'cboltz' + pending: + - 'https://github.com/saltstack-formulas/elasticsearch-formula/pull/36' +grains: + namespace: 'tampakrap' +keepalived: {} +limits: + namespace: 'ryancurrah' +locale: {} +mysql: {} +ntp: {} +openldap: {} +openssh: {} +powerdns: {} +salt: {} +sqlite: + namespace: 'tampakrap' +sssd: + original_namespace: 'Spark-Networks' + namespace: 'tampakrap' + pending: + - 'https://github.com/Spark-Networks/salt-sssd-formula/pull/1' + prefix: 'salt-' +sudoers: {} +timezone: {} +users: {} +zypper: + namespace: 'tampakrap' diff --git a/bin/encrypt_pillar.sh b/bin/encrypt_pillar.sh index 35d0d84..71c1061 100755 --- a/bin/encrypt_pillar.sh +++ b/bin/encrypt_pillar.sh @@ -8,7 +8,7 @@ help() { [[ $1 == '--help' ]] && help && exit -while getopts s:v:h arg; do +while getopts h arg; do case ${arg} in h) help && exit ;; *) help && exit 1 ;; diff --git a/bin/get_formulas.py b/bin/get_formulas.py new file mode 100755 index 0000000..f7c314b --- /dev/null +++ b/bin/get_formulas.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 + +# For description and usage, see the argparse options at the end of the file + +import argparse +import os +import yaml + + +def clone(DEST, SYMLINK=False): + def use_git_to_clone_or_pull_repo(): + # pygit2 is not available for python3 in Leap, use plain git instead + + import subprocess + + if not os.path.exists(DEST): + os.mkdir(DEST) + if os.path.isdir(FULL_PATH): + subprocess.Popen(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + subprocess.Popen(['git', 'clone', url, FULL_PATH], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def use_pygit2_to_clone_or_pull_repo(): + import pygit2 + + if os.path.isdir(FULL_PATH): + repo = pygit2.Repository(FULL_PATH) + repo.checkout('HEAD') + else: + pygit2.clone_repository(url, FULL_PATH, bare=False) + + for formula, data in FORMULAS.items(): + namespace = data.get('namespace', 'saltstack-formulas') + prefix = data.get('prefix', '') + url = 'https://github.com/%s/%s%s-formula' % (namespace, prefix, formula) + FULL_PATH = '%s/%s-formula' % (DEST, formula) + use_git_to_clone_or_pull_repo() + if SYMLINK: + os.symlink('%s/%s' % (FULL_PATH, formula), '/srv/salt/%s' % formula) + + +with open('FORMULAS.yaml', 'r') as f: + FORMULAS = yaml.load(f) + +parser = argparse.ArgumentParser(description='Loads the formulas from FORMULAS.yaml and optionally clones them in a specified destination. Optionally it can also create a symlink from the cloned path to /srv/salt, useful for the CI worker.') +parser.add_argument('-c', '--clone', nargs=1, help='Clones the formulas to a specified destination that is passed as option to this argument.') +parser.add_argument('-s', '--symlink', action='store_true', help='Creates symlink from the specified destination to /srv/salt.') +args = parser.parse_args() + +if args.clone: + clone(args.clone[0], args.symlink) diff --git a/bin/get_roles.py b/bin/get_roles.py new file mode 100755 index 0000000..3534fc7 --- /dev/null +++ b/bin/get_roles.py @@ -0,0 +1,47 @@ +#!/usr/bin/python3 + +# For description and usage, see the argparse options at the end of the file + +import argparse +import yaml +import os + + +def get_roles(with_base=False): + roles = [] + if with_base: + roles.append('base') + + for sls in os.listdir('pillar/id'): + with open("pillar/id/%s" % sls) as f: + try: + _roles = yaml.load(f)['grains']['roles'] + except KeyError: + continue + for item in _roles: + roles.append(item) + + roles = sorted(set(roles)) + return roles + + +def print_roles(): + parser = argparse.ArgumentParser('Collects all the roles that are assigned to a minion, and returns them as a python array, a yaml list or a plain list (parsable by bash)') + parser.add_argument('-p', '--python', action='store_true', help='Prints the roles as a python array') + parser.add_argument('-y', '--yaml', action='store_true', help='Prints the roles as a yaml array') + parser.add_argument('--with-base', action='store_true', help='Include the base role at the results') + args = parser.parse_args() + + roles = get_roles(with_base=args.with_base) + if args.python: + print(roles) + elif args.yaml: + print('roles:') + for role in roles: + print(' - %s' % role) + else: + print(' '.join(roles)) + + +if __name__ == "__main__": + print_roles() diff --git a/bin/test_roles.py b/bin/test_roles.py index aef27f5..d899178 100755 --- a/bin/test_roles.py +++ b/bin/test_roles.py @@ -1,32 +1,21 @@ #!/usr/bin/python3 -# Collects all the assigned roles of all the minions, and checks if the -# declared {salt,pillar}/role/*.sls files actually match a minion-assigned role +# Checks if the declared {salt,pillar}/role/*.sls files actually match an +# assigned role to a minion -import yaml import os import sys +from get_roles import get_roles -all_roles = ['base'] status = 0 -for sls in os.listdir('pillar/id'): - with open("pillar/id/%s" % sls) as f: - try: - _roles = yaml.load(f)['grains']['roles'] - except KeyError: - continue - - for item in _roles: - all_roles.append(item) - -all_roles = sorted(set(all_roles)) +roles = get_roles(with_base=True) for directory in ['salt', 'pillar']: for sls in os.listdir('%s/role' % directory): if sls.endswith('.sls'): with open('%s/role/%s' % (directory, sls)) as f: - if sls.split('.sls')[0] not in all_roles: + if sls.split('.sls')[0] not in roles: print ('%s/role/%s not in roles' % (directory, sls)) status = 1 diff --git a/bin/test_show_highstate.py b/bin/test_show_highstate.py new file mode 100755 index 0000000..bf70ded --- /dev/null +++ b/bin/test_show_highstate.py @@ -0,0 +1,19 @@ +#!/bin/bash + +# Runs state.show_highstate using all localized grains' combinations + +set -e + +RUN_TEST="salt-call --local --retcode-passthrough state.show_highstate" +ROLES=$(bin/get_roles.py --yaml) + +echo 'domain: infra.opensuse.org' > /etc/salt/grains +printf "city:\ncountry:\nsalt_cluster: opensuse\nvirt_cluster:\n$ROLES" > pillar/id/${HOSTNAME}.sls + +sed -i -e 's/\(city:\).*/\1 nuremberg/' -e 's/\(country:\).*/\1 de/' -e 's/\(virt_cluster:\).*/\1 atreju/' pillar/id/${HOSTNAME}.sls +$RUN_TEST > /dev/null +echo "PASSED: country: de" + +sed -i -e 's/\(city:\).*/\1 provo/' -e 's/\(country:\).*/\1 us/' -e 's/\(virt_cluster:\).*/\1 bryce/' pillar/id/${HOSTNAME}.sls +$RUN_TEST > /dev/null +echo "PASSED: country: us"