diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2103c6f..6a682aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,37 +1,22 @@
 stages:
+  - validate
   - test
   - deploy
 
-test_extension:
-  stage: test
-  script: bin/test_extension.sh
-  tags:
-    - docker
-
-test_secrets:
-  stage: test
-  script: bin/test_secrets.sh
-  tags:
-    - docker
-
-test_roles:
-  stage: test
-  before_script:
-    - zypper -qn in --no-recommends python3 python3-PyYAML
-  script: bin/test_roles.py
-  tags:
-    - docker
-
-test_custom_grains:
-  stage: test
+validate:
+  stage: validate
+  variables:
+    # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs
+    GIT_SSL_NO_VERIFY: 'true'
   before_script:
-    - zypper -qn in --no-recommends python3 python3-PyYAML
-  script: bin/test_custom_grains.py
+    - bin/prepare_test_highstate_env.sh -s
+    - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production
+  script: bin/test_validate.sh
   tags:
     - docker
 
-test_show_highstate_against_upstream_formulas:
-  stage: test
+validate_show_highstate_against_upstream_formulas:
+  stage: validate
   before_script:
     - bin/prepare_test_highstate_env.sh -s
     - bin/get_formulas.py -c -d /srv/formula -s
@@ -42,18 +27,6 @@ test_show_highstate_against_upstream_formulas:
   tags:
     - docker
 
-test_show_highstate_against_forked_formulas:
-  stage: test
-  variables:
-    # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs
-    GIT_SSL_NO_VERIFY: 'true'
-  before_script:
-    - bin/prepare_test_highstate_env.sh -s
-    - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production
-  script: bin/test_show_highstate.sh
-  tags:
-    - docker
-
 test_nginx:
   stage: test
   variables:
diff --git a/bin/run_tests_locally.sh b/bin/run_tests_locally.sh
index 2618f4c..93366a0 100755
--- a/bin/run_tests_locally.sh
+++ b/bin/run_tests_locally.sh
@@ -29,11 +29,8 @@ SALT_DIRS=(
     /var/cache/salt
 )
 
-bin/test_extension.sh
-bin/test_secrets.sh
-bin/test_roles.py
-bin/test_custom_grains.py
-bin/prepare_test_show_highstate_env.sh
+# Prepare env
+bin/prepare_test_highstate_env.sh
 bin/get_formulas.py --destination $DESTINATION --clone --symlink --use-pygit2 --update opensuse \
     --add-remote opensuse no_prefix gitlab@gitlab.infra.opensuse.org: saltstack-formulas
 for dir in ${SALT_DIRS[@]}; do
@@ -41,8 +38,10 @@ for dir in ${SALT_DIRS[@]}; do
 done
 echo "virtual: kvm" >> /etc/salt/grains
 ln -s ~/.gnupg /etc/salt/gpgkeys
+
+# Run tests
 echo "Running against upstream formulas"
-LC_ALL=C bin/test_show_highstate.sh
+LC_ALL=C bin/test_validate.sh
 bin/get_formulas.py --destination $DESTINATION --checkout opensuse/production
 echo "Running against forked formulas"
 LC_ALL=C bin/test_show_highstate.sh
diff --git a/bin/test_validate.sh b/bin/test_validate.sh
new file mode 100755
index 0000000..d762bec
--- /dev/null
+++ b/bin/test_validate.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run various code validation/syntax checks
+
+TESTS=(
+    extension.sh
+    secrets.sh
+    roles.py
+    custom_grains.py
+    show_highstate.sh
+)
+
+for _test in ${TESTS[@]}; do
+    bin/test_${_test} || STATUS=1
+done
+
+exit $STATUS