diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8f60fd4..d517369 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,6 +54,18 @@ test_show_highstate_against_forked_formulas:
   tags:
     - docker
 
+test_nginx:
+  stage: test
+  variables:
+    # TODO: get rid of GIT_SSL_NO_VERIFY as soon as we switch to letsencrypt wildcard certs
+    GIT_SSL_NO_VERIFY: 'true'
+  before_script:
+    - bin/prepare_test_show_highstate_env.sh -s
+    - bin/get_formulas.py -c -d /srv/formula -s --clone-from 'https://gitlab.infra.opensuse.org/saltstack-formulas' --clone-branch production
+  script: bin/test_nginx.sh
+  tags:
+    - docker
+
 deploy_job:
   stage: deploy
   script: sudo salt-call event.fire_master $CI_DEPLOY_PASSWORD salt/fileserver/gitfs/update
diff --git a/bin/prepare_test_show_highstate_env.sh b/bin/prepare_test_show_highstate_env.sh
index 2ddef0e..056b62f 100755
--- a/bin/prepare_test_show_highstate_env.sh
+++ b/bin/prepare_test_show_highstate_env.sh
@@ -39,5 +39,5 @@ $SUDO ln -s $PWD/pillar /srv/pillar
 ID=$(hostname -f)
 ROLES=$(bin/get_roles.py -o yaml)
 printf "city:\ncountry:\ndomain: infra.opensuse.org\nosfullname:\nosmajorrelease:\nosrelease_info:\n$ROLES\nsalt_cluster: opensuse\nvirt_cluster:\n" | $SUDO tee /etc/salt/grains > /dev/null
-touch pillar/id/${ID//./_}.sls
+printf "nothing: nothing" | $SUDO tee pillar/id/${ID//./_}.sls > /dev/null
 [[ -n $STRIP_SECRETS ]] && sed -i -e "s#\- secrets\..*#- id.${ID//./_}#g" $(grep -lr "\- secrets\." pillar)
diff --git a/bin/test_nginx.sh b/bin/test_nginx.sh
new file mode 100755
index 0000000..6c7dde3
--- /dev/null
+++ b/bin/test_nginx.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Validate the salt-generated nginx configs
+
+if [[ $(whoami) != 'root' ]]; then
+    if [[ -f /usr/bin/sudo ]]; then
+        SUDO='/usr/bin/sudo'
+    else
+        echo 'Please install sudo first, or run this script as root'
+        exit 1
+    fi
+fi
+
+reset_nginx() {
+    $SUDO rm -rf /etc/nginx/vhosts.d/*
+    printf "roles:\n- $role" | $SUDO tee /etc/salt/grains > /dev/null
+}
+
+WEB_ROLES=( $(bin/get_roles.py | grep web_) )
+
+for role in ${WEB_ROLES[@]}; do
+    if grep nginx salt/role/$role.sls > /dev/null; then
+        echo "Testing role: $role"
+        reset_nginx
+        $SUDO salt-call --local -l quiet state.apply role.$role > /dev/null
+        if $(nginx -tq); then
+            echo 'PASSED'
+        else
+            STATUS=1
+        fi
+    fi
+done
+
+exit $STATUS