diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aee2096..fc638a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,16 +6,51 @@ stages: validate: stage: validate before_script: - - bin/prepare_test_env.sh -g - - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production + - bin/prepare_test_env.sh -i opensuse script: bin/test_validate.sh tags: - docker -validate_show_highstate_against_upstream_formulas: +leap15.0_show_highstate: + stage: validate + before_script: + - bin/prepare_test_env.sh -i opensuse -g + - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production + script: bin/test_show_highstate.sh + tags: + - docker + +leap42.3_show_highstate: + stage: validate + before_script: + - bin/prepare_test_env.sh -i opensuse -g -o Leap,42,3 + - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production + script: bin/test_show_highstate.sh + tags: + - docker + +sle12sp3_show_highstate: + stage: validate + before_script: + - bin/prepare_test_env.sh -i opensuse -g -o SLES,12,3 + - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production + script: bin/test_show_highstate.sh + tags: + - docker + +sle11sp4_show_highstate: + stage: validate + before_script: + - bin/prepare_test_env.sh -i opensuse -g -o SLES,11,4 + - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production + script: bin/test_show_highstate.sh + tags: + - docker + +upstream_formulas_show_highstate: stage: validate before_script: - - bin/prepare_test_env.sh -g + - bin/prepare_test_env.sh -i opensuse -g - bin/get_formulas.py -c -d /srv/formula -s script: bin/test_show_highstate.sh allow_failure: true @@ -27,7 +62,7 @@ validate_show_highstate_against_upstream_formulas: test_nginx: stage: test before_script: - - bin/prepare_test_env.sh -g -p nginx + - bin/prepare_test_env.sh -i opensuse -g -p nginx - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production -f nginx script: bin/test_nginx.sh tags: diff --git a/bin/get_valid_custom_grains.py b/bin/get_valid_custom_grains.py index d83b160..4766df5 100755 --- a/bin/get_valid_custom_grains.py +++ b/bin/get_valid_custom_grains.py @@ -1,5 +1,8 @@ #!/usr/bin/python3 +# For description and usage, see the argparse options at the end of the file + +import argparse import yaml @@ -18,6 +21,35 @@ def get_all_valid_localized_grains(): return get_valid_custom_grains()['localized'] +def get_virt_cluster_only_physical(): + virt_cluster_only_physical = '' + try: + virt_cluster_only_physical = get_valid_custom_grains()['virt_cluster_only_physical'] + except KeyError: + pass + print('\n'.join(virt_cluster_only_physical)) + + +def get_all_valid_domains(country): + all_valid_domains = get_all_valid_localized_grains()[country]['domains'] + if type(all_valid_domains) == str: + # convert to list + all_valid_domains = [all_valid_domains] + print('\n'.join(all_valid_domains)) + + +def get_default_domain(country): + print(get_all_valid_localized_grains()[country]['default_domain']) + + +def get_default_virt_cluster(): + results = [] + all_valid_localized_grains = get_all_valid_localized_grains() + for country, items in all_valid_localized_grains.items(): + results.append('%s,%s,%s' % (country, items['city'], items['default_virt_cluster'])) + print('\n'.join(results)) + + def print_valid_localized_grains(): results = [] all_valid_localized_grains = get_all_valid_localized_grains() @@ -34,4 +66,20 @@ def print_valid_localized_grains(): if __name__ == "__main__": - print_valid_localized_grains() + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description='Loads the pillar/valid_custom_grains.py and returns a list of valid custom grains in the form of "country,city,virt_cluster".') + parser.add_argument('-p', action='store_true', help='Returns a list of physical machines that do not host any salt-managed VMs.') + parser.add_argument('-d', nargs=1, help='Returns a list of the valid domains of a location.') + parser.add_argument('--default-domain', nargs=1, help='Returns the default domain of a location.') + parser.add_argument('-v', action='store_true', help='Returns the valid custom grains, but only displaying once each country with their default virt_cluster.') + args = parser.parse_args() + + if args.p: + get_virt_cluster_only_physical() + elif args.d: + get_all_valid_domains(args.d[0]) + elif args.default_domain: + get_default_domain(args.default_domain[0]) + elif args.v: + get_default_virt_cluster() + else: + print_valid_localized_grains() diff --git a/bin/prepare_test_env.sh b/bin/prepare_test_env.sh index eb2133d..4773cac 100755 --- a/bin/prepare_test_env.sh +++ b/bin/prepare_test_env.sh @@ -19,18 +19,24 @@ help() { echo "Arguments:" echo echo "-p Comma-separated list of additional packages to be installed" - echo "-g Make preparation for show_highstate" - echo "-s Include secrets files (disabed because CI runner can't decrypt them due to lack of GPG key)" + echo "-i Choose gitlab instance. Choices: opensuse, suse" + echo "-o OPTIONAL: Specify different OS. Examples: \"Leap,42,3\", \"SLES,12,3\"" + echo "-g OPTIONAL: Make preparation for show_highstate" + echo "-s OPTIONAL: Include secrets files (disabed because CI runner can't decrypt them due to lack of GPG key)" echo } [[ $1 == '--help' ]] && help && exit +#source /etc/os-release SECRETS="False" +#REPO_URL=${PRETTY_NAME// /_} -while getopts p:gsh arg; do +while getopts p:i:o:gsh arg; do case ${arg} in p) PKG=(${OPTARG//,/ }) ;; + i) INSTANCE=${OPTARG} ;; + o) OS=(${OPTARG//,/ }) ;; g) HIGHSTATE=1 ;; s) SECRETS="True" ;; h) help && exit ;; @@ -38,15 +44,31 @@ while getopts p:gsh arg; do esac done +[[ -z $INSTANCE ]] && help && exit 1 [[ -n $HIGHSTATE ]] && HIGHSTATE_PKGS=( git python3-PyYAML ) -$SUDO zypper -qn in --no-recommends salt python3-pygit2 ca-certificates-freeipa-opensuse ${HIGHSTATE_PKGS[@]} ${PKG[@]} +if [[ $INSTANCE == 'opensuse' ]]; then + DOMAIN='infra.opensuse.org' + SALT_CLUSTER='opensuse' + VIRT_CLUSTER='atreju' + #$SUDO zypper -q ar -p 100 -fG http://download.opensuse.org/repositories/openSUSE:/infrastructure/${REPO_URL}/ openSUSE:infrastructure > /dev/null 2>&1 + CERT_PKG='ca-certificates-freeipa-opensuse' +elif [[ $INSTANCE == 'suse' ]]; then + DOMAIN='suse.de' + SALT_CLUSTER='suse_internal' + VIRT_CLUSTER='morla' + #$SUDO zypper -q ar -p 100 -fG http://download.nue.suse.com/ibs/SUSE:/CA/${REPO_URL}/ SUSE:CA > /dev/null 2>&1 + CERT_PKG='ca-certificates-suse' +fi + +#$SUDO zypper -qn in --no-recommends salt python3-pygit2 $CERT_PKG ${HIGHSTATE_PKGS[@]} ${PKG[@]} $SUDO rm -rf /srv/{salt,pillar} $SUDO ln -s $PWD/salt /srv/salt $SUDO ln -s $PWD/pillar /srv/pillar ID=$(hostname -f) -printf "grains:\n city: nuremberg\n country: de\n hostusage: test\n salt_cluster: opensuse\n virt_cluster: atreju\n" > pillar/id/${ID//./_}.sls +printf "grains:\n city: nuremberg\n country: de\n hostusage: test\n salt_cluster: $SALT_CLUSTER\n virt_cluster: $VIRT_CLUSTER\n" > pillar/id/${ID//./_}.sls if [[ -n $HIGHSTATE ]]; then ROLES=$(bin/get_roles.py -o yaml) - printf "city:\ncountry:\ndomain: infra.opensuse.org\ninclude_secrets: $SECRETS\nosfullname:\nosmajorrelease:\nosrelease_info:\n$ROLES\nsalt_cluster: opensuse\nvirt_cluster:\nvirtual:\n" > /etc/salt/grains + [[ -n $OS ]] && OS_GRAINS="osfullname: ${OS[0]}\nosmajorrelease: ${OS[1]}\nosrelease_info: [${OS[1]}, ${OS[2]}]\n" + printf "city:\ncountry:\ndomain: $DOMAIN\ninclude_secrets: $SECRETS\n$OS_GRAINS$ROLES\nsalt_cluster: $SALT_CLUSTER\nvirt_cluster:\nvirtual:\n" > /etc/salt/grains fi diff --git a/bin/test_custom_grains.py b/bin/test_custom_grains.py index ecf0270..62e48f8 100755 --- a/bin/test_custom_grains.py +++ b/bin/test_custom_grains.py @@ -53,6 +53,8 @@ for sls in all_ids: try: valid_localized_grains = all_valid_localized_grains[mygrains['country']] + for ignored_key in ['domains', 'default_domain', 'default_virt_cluster']: + valid_localized_grains.pop(ignored_key, None) for key, valid_values in valid_localized_grains.items(): status = test_custom_grain(mygrains, sls, key, valid_values, status) except KeyError: diff --git a/bin/test_show_highstate.sh b/bin/test_show_highstate.sh index 45ca046..f7e645d 100755 --- a/bin/test_show_highstate.sh +++ b/bin/test_show_highstate.sh @@ -17,36 +17,54 @@ RUN_TEST="salt-call --local --retcode-passthrough state.show_highstate" ALL_VIRTUAL=( kvm ) -ALL_OS=( - Leap,15,0 - Leap,42,3 - SLES,11,4 - SLES,12,3 -) -ALL_LOCATIONS=( $(bin/get_valid_custom_grains.py) ) +PHYSICAL_ONLY_VIRT_CLUSTER=( $(bin/get_valid_custom_grains.py -p) ) write_grains() { - $SUDO sed -i -e "s/\(city:\).*/\1 $2/" -e "s/\(country:\).*/\1 $1/" -e "s/\(osfullname:\).*/\1 $4/" -e "s/\(osmajorrelease:\).*/\1 $5/" \ - -e "s/\(osrelease_info:\).*/\1 [$5, $6]/" -e "s/\(virt_cluster:\).*/\1 $3/" -e "s/\(virtual:\).*/\1 $7/" /etc/salt/grains - echo_INFO "Grains: osfullname: $4, osmajorrelease: $5, city: $2, country: $1, virt_cluster: $3, virtual: $7" + $SUDO sed -i -e "s/\(city:\).*/\1 $2/" -e "s/\(country:\).*/\1 $1/" -e "s/\(domain:\).*/\1 $5/" -e "s/\(virt_cluster:\).*/\1 $3/" -e "s/\(virtual:\).*/\1 $4/" /etc/salt/grains + echo_INFO "Grains: city: $2, country: $1, domain: $5, virt_cluster: $3, virtual: $4" +} + +show_highstate() { + write_grains $country $city $virt_cluster $virtual $domain + $RUN_TEST > /dev/null + _STATUS=$? + # We ignore exit code 2 as it means that an empty file is produced + # See https://github.com/saltstack/salt/issues/39172 + if [[ $_STATUS -eq 0 ]] || [[ $_STATUS -eq 2 ]]; then + echo_PASSED + else + echo_FAILED + STATUS=1 + fi + echo } -for os in ${ALL_OS[@]}; do - for location in ${ALL_LOCATIONS[@]}; do - for virtual in ${ALL_VIRTUAL[@]}; do - write_grains ${location//,/ } ${os//,/ } ${virtual} - $RUN_TEST > /dev/null - _STATUS=$(echo $?) - # We ignore exit code 2 as it means that an empty file is produced - # See https://github.com/saltstack/salt/issues/39172 - if [[ $_STATUS -eq 0 ]] || [[ $_STATUS -eq 2 ]]; then - echo_PASSED - else - echo_FAILED - STATUS=1 - fi - echo - done +ALL_LOCATIONS=( $(bin/get_valid_custom_grains.py) ) +for location in ${ALL_LOCATIONS[@]}; do + LOCATION=(${location//,/ }) + country=${LOCATION[0]} + city=${LOCATION[1]} + virt_cluster=${LOCATION[2]} + domain=$(bin/get_valid_custom_grains.py --default-domain $country) + for virtual in ${ALL_VIRTUAL[@]}; do + if [[ $virtual == 'kvm' ]] && [[ ${PHYSICAL_ONLY_VIRT_CLUSTER[@]} =~ $virt_cluster ]]; then + continue + fi + show_highstate + done +done + +ALL_LOCATIONS=( $(bin/get_valid_custom_grains.py -v) ) +for location in ${ALL_LOCATIONS[@]}; do + LOCATION=(${location//,/ }) + country=${LOCATION[0]} + city=${LOCATION[1]} + virt_cluster=${LOCATION[2]} + default_domain=$(bin/get_valid_custom_grains.py --default-domain $country) + virtual='kvm' + DOMAINS=( $(bin/get_valid_custom_grains.py -d $country) ) + for domain in ${DOMAINS[@]}; do + [[ $domain == $default_domain ]] || show_highstate done done diff --git a/bin/test_validate.sh b/bin/test_validate.sh index bc456a2..edaebf5 100755 --- a/bin/test_validate.sh +++ b/bin/test_validate.sh @@ -10,7 +10,6 @@ TESTS=( secrets.sh roles.py custom_grains.py - show_highstate.sh ) for _test in ${TESTS[@]}; do diff --git a/pillar/valid_custom_grains.yaml b/pillar/valid_custom_grains.yaml index 1494c7b..9633f03 100644 --- a/pillar/valid_custom_grains.yaml +++ b/pillar/valid_custom_grains.yaml @@ -5,7 +5,13 @@ global: localized: de: city: nuremberg + default_domain: infra.opensuse.org + default_virt_cluster: atreju + domains: infra.opensuse.org virt_cluster: atreju us: city: provo + default_domain: infra.opensuse.org + default_virt_cluster: bryce + domains: infra.opensuse.org virt_cluster: bryce