From 9d40b0934f16b8edcff3d53512bce531e650642b Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Jan 13 2024 23:51:21 +0000 Subject: Add style test for common file types Add linting script and run it against files with following extensions: - .jinja, .j2 - .py - .sh - .sls - .yaml This is an effort to establish a common coding style and uniform datasets in our repository. Signed-off-by: Georg Pfuetzenreuter --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5974c0..947d5ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,12 @@ stages: paths: - '*.txt' +lint: + extends: + - .test_common + image: registry.opensuse.org/opensuse/infrastructure/containers_tumbleweed/heroes-salt-validation:latest + script: bin/lint.sh + validate: extends: - .test_common diff --git a/bin/lint.sh b/bin/lint.sh new file mode 100755 index 0000000..fd5144f --- /dev/null +++ b/bin/lint.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Script to lint and style check our code +# Copyright (C) 2024 Georg Pfuetzenreuter +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -Cfu +EXIT=0 + +. bin/get_colors.sh + +echo_INFO 'Linting Jinja files ...' +# j2lint does not support passing exclusions through a configuration file +j2lint_args=( + jinja-statements-delimiter # allow for {%- (with the hyphen for reduced empty lines) + jinja-statements-indentation # allow for custom indentation + jinja-variable-lower-case # https://github.com/aristanetworks/j2lint/issues/85 (should be removed when a solution is found) + operator-enclosed-by-spaces # allow for "foo|bar" without spaces + jinja-statements-no-tabs # false positive in configuration templates using tabs + single-statement-per-line # allow for multiple statements in one line: {%- foo -%}{%- bar -%} +) +find . -not -path ./salt/files/\* -type f \( -name '*.j2' -o -name '*.jinja' \) \ + -exec j2lint -i "${j2lint_args[@]}" -- {} + +STATUS_JINJA="$?" +j2lint_args+=( + single-statement-per-line # false positive in Libvirt XML templates +) +find salt/files -type f \( -name '*.j2' -o -name '*.jinja' \) \ + -exec j2lint -i "${j2lint_args[@]}" -- {} + +STATUS_JINJA="$?" + + +echo_INFO 'Linting Python files ...' +ruff . +STATUS_PYTHON="$?" + + +echo_INFO 'Linting Python files in profiles ...' +TMP_PY="$(mktemp -dp /dev/shm)" || exit 1 +export TMP_PY +find salt/profile -type f -name '*.py' \ + -exec sh -c ' + FILE="$1" + DIR="$(dirname $FILE)" + mkdir -p "$TMP_PY/$DIR" + grep -Ev "{(%|{|#)" "$FILE" > "$TMP_PY/$FILE" + ' x {} \; +pushd "$TMP_PY" >/dev/null || EXIT=1 +ruff --config "$OLDPWD"/ruff.toml . +STATUS_PYTHON_PROFILE="$?" +popd >/dev/null || EXIT=1 +rm -r "$TMP_PY" + + +echo_INFO 'Linting Shell files ...' +# TODO: Include all optional suggestions, except for require-double-brackets and require-variable-braces (-o all -e SC2250,SC2292) +find . -not -path ./t/\* -type f -name '*.sh' \ + -exec shellcheck -x {} + +STATUS_SHELL="$?" + + +echo_INFO 'Linting SLS files ...' +find . -type f -name '*.sls' \ + -exec salt-lint {} + +STATUS_SLS="$?" + + +echo_INFO 'Linting YAML files ...' +find pillar/ -type f -name '*.yaml' \ + -exec yamllint {} + +STATUS_YAML="$?" + + +echo +echo '===================' +echo '===== SUMMARY =====' +echo '===================' +echo + +if [ "$STATUS_JINJA" = 1 ] +then + echo '--> Jinja: FAIL' + echo 'Please check the suggested Jinja formatting changes. Note that these are suggestions, and not all cases are considered. If the implementation of a particular suggestion is unreasonable, please discuss whether the rule should be added to the ignore list.' + EXIT=5 +else + echo '--> Jinja: PASS' +fi +echo + +if [ "$STATUS_PYTHON" = 1 ] || [ "$STATUS_PYTHON_PROFILE" = 1 ] +then + echo '--> Python: FAIL' + echo 'Please reformat the problematic Python files to follow the PEP 8 style guide.' + echo 'Reference: https://peps.python.org/pep-0008/.' + # shellcheck disable=SC2016 + echo 'Tip: Install `ruff` on your workstation, and use `ruff --fix --unsafe-fixes ...` to have the tool automatically implement some of its suggestions.' + EXIT=5 +else + echo '--> Python: PASS' +fi +echo + +if [ "$STATUS_SHELL" = 1 ] +then + echo '--> Shell: FAIL' + echo 'Please try to apply the ShellCheck suggestions. Note that the Shebang is used to determine which suggestions should be applied.' + echo 'If a particular suggestion is not implementable, a comment based override should be declared before the problematic line.' + EXIT=5 +else + echo '--> Shell: PASS' +fi +echo + +if [ "$STATUS_SLS" = 1 ] +then + echo '--> SLS: FAIL' + echo 'Please check and apply the salt-lint suggestions.' + EXIT=5 +else + echo '--> SLS: PASS' +fi +echo + +if [ "$STATUS_YAML" = 1 ] +then + echo '--> YAML: FAIL' + echo 'Please check and apply the yamllint suggestions.' + echo 'In most cases, these should be very reasonable and easy to implement. In rare cases where it is not feasible, a comment based exclude in the line before the problematic one can be placed. This however is not possible for files run through sort_yaml.py.' + EXIT=5 +else + echo '--> YAML: PASS' +fi +echo + +if [ "$EXIT" = 5 ] +then + echo 'Please check the linter suggestions mentioned above and contribute to a common coding style. Thank you!' +elif [ "$EXIT" = 1 ] +then + echo 'Execution error.' +fi + +exit "$EXIT" diff --git a/bin/test_infra_data.sh b/bin/test_infra_data.sh index c13c2db..32b53a2 100755 --- a/bin/test_infra_data.sh +++ b/bin/test_infra_data.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Script to lint and validate openSUSE infrastructure data +# Script to validate openSUSE infrastructure data # Copyright (C) 2023 Georg Pfuetzenreuter # # This program is free software: you can redistribute it and/or modify @@ -19,11 +19,6 @@ set -Cu INFRADIR='pillar/infra/' -echo 'test_infra_data --> Linting ...' -yamllint -s "$INFRADIR"*.yaml -RESULT_YAML="$?" -echo - echo 'test_infra_data --> Validating ...' bin/test_infra_data.py RESULT_SCHEMA="$?" @@ -40,9 +35,9 @@ RESULT_SORT="$?" rm "$INFRADIR"hosts.sorted.yaml echo -echo "test_infra_data --> Results: YAML -> $RESULT_YAML, SCHEMA -> $RESULT_SCHEMA, SORT -> $RESULT_SORT" +echo "test_infra_data --> SCHEMA -> $RESULT_SCHEMA, SORT -> $RESULT_SORT" -if [ "$RESULT_YAML" = 0 ] && [ "$RESULT_SCHEMA" = 0 ] && [ "$RESULT_SORT" = 0 ] +if [ "$RESULT_SCHEMA" = 0 ] && [ "$RESULT_SORT" = 0 ] then exit 0 elif [ "$RESULT_SORT" = 1 ] # 123 in case of xargs