From b4fa0db4b838beb43a16b2a2cb9d9e358358eb7a Mon Sep 17 00:00:00 2001
From: Theo Chatzimichos <tampakrap@opensuse.org>
Date: Jan 21 2018 23:58:50 +0000
Subject: group the tests to run them faster:


- introduce stage "validate" that runs code validation/syntax checking
- introduce script bin/test_validate.sh that runs all the validation
  tests that are not allowed to fail. This way only one container is
  created reducing the time of starting/stopping for the next test. No
  real need for separate container is needed, as the environment is the
  same for all of those tests
- the show_highstate against upstream formulas test job is being run
  separately as it is allowed to fail, and we don't want it to run on
  production branch (also because there is a change of the testing env)
- the rest of the tests (only test_nginx for now) are in a test stage,
  which means they will run only if the "validate" stage finishes
  successfully (we don't want to test the generated configs if we know
  that our code is wrong, so less waiting for the test runs)

---

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2103c6f..6a682aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,37 +1,22 @@
 stages:
+  - validate
   - test
   - deploy
 
-test_extension:
-  stage: test
-  script: bin/test_extension.sh
-  tags:
-    - docker
-
-test_secrets:
-  stage: test
-  script: bin/test_secrets.sh
-  tags:
-    - docker
-
-test_roles:
-  stage: test
-  before_script:
-    - zypper -qn in --no-recommends python3 python3-PyYAML
-  script: bin/test_roles.py
-  tags:
-    - docker
-
-test_custom_grains:
-  stage: test
+validate:
+  stage: validate
+  variables:
+    # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs
+    GIT_SSL_NO_VERIFY: 'true'
   before_script:
-    - zypper -qn in --no-recommends python3 python3-PyYAML
-  script: bin/test_custom_grains.py
+    - bin/prepare_test_highstate_env.sh -s
+    - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production
+  script: bin/test_validate.sh
   tags:
     - docker
 
-test_show_highstate_against_upstream_formulas:
-  stage: test
+validate_show_highstate_against_upstream_formulas:
+  stage: validate
   before_script:
     - bin/prepare_test_highstate_env.sh -s
     - bin/get_formulas.py -c -d /srv/formula -s
@@ -42,18 +27,6 @@ test_show_highstate_against_upstream_formulas:
   tags:
     - docker
 
-test_show_highstate_against_forked_formulas:
-  stage: test
-  variables:
-    # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs
-    GIT_SSL_NO_VERIFY: 'true'
-  before_script:
-    - bin/prepare_test_highstate_env.sh -s
-    - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production
-  script: bin/test_show_highstate.sh
-  tags:
-    - docker
-
 test_nginx:
   stage: test
   variables:
diff --git a/bin/run_tests_locally.sh b/bin/run_tests_locally.sh
index 2618f4c..93366a0 100755
--- a/bin/run_tests_locally.sh
+++ b/bin/run_tests_locally.sh
@@ -29,11 +29,8 @@ SALT_DIRS=(
     /var/cache/salt
 )
 
-bin/test_extension.sh
-bin/test_secrets.sh
-bin/test_roles.py
-bin/test_custom_grains.py
-bin/prepare_test_show_highstate_env.sh
+# Prepare env
+bin/prepare_test_highstate_env.sh
 bin/get_formulas.py --destination $DESTINATION --clone --symlink --use-pygit2 --update opensuse \
     --add-remote opensuse no_prefix gitlab@gitlab.infra.opensuse.org: saltstack-formulas
 for dir in ${SALT_DIRS[@]}; do
@@ -41,8 +38,10 @@ for dir in ${SALT_DIRS[@]}; do
 done
 echo "virtual: kvm" >> /etc/salt/grains
 ln -s ~/.gnupg /etc/salt/gpgkeys
+
+# Run tests
 echo "Running against upstream formulas"
-LC_ALL=C bin/test_show_highstate.sh
+LC_ALL=C bin/test_validate.sh
 bin/get_formulas.py --destination $DESTINATION --checkout opensuse/production
 echo "Running against forked formulas"
 LC_ALL=C bin/test_show_highstate.sh
diff --git a/bin/test_validate.sh b/bin/test_validate.sh
new file mode 100755
index 0000000..d762bec
--- /dev/null
+++ b/bin/test_validate.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run various code validation/syntax checks
+
+TESTS=(
+    extension.sh
+    secrets.sh
+    roles.py
+    custom_grains.py
+    show_highstate.sh
+)
+
+for _test in ${TESTS[@]}; do
+    bin/test_${_test} || STATUS=1
+done
+
+exit $STATUS