diff --git a/pillar/id/pinot_infra_opensuse_org.sls b/pillar/id/pinot_infra_opensuse_org.sls index b20e154..0201616 100644 --- a/pillar/id/pinot_infra_opensuse_org.sls +++ b/pillar/id/pinot_infra_opensuse_org.sls @@ -5,6 +5,7 @@ grains: - countdown.o.o roles: - countdown + - documentation - mail_reminder reboot_safe: yes salt_cluster: opensuse diff --git a/pillar/role/documentation.sls b/pillar/role/documentation.sls new file mode 100644 index 0000000..1bb6806 --- /dev/null +++ b/pillar/role/documentation.sls @@ -0,0 +1,16 @@ +include: + - role.common.nginx + +nginx: + ng: + servers: + managed: + doc.opensuse.org.conf: + config: + - server: + - server_name: doc.opensuse.org + - listen: + - 80 + - location /: + - root: /srv/www/vhosts/doc.opensuse.org/ + enabled: True diff --git a/salt/profile/documentation/files/releasenotes b/salt/profile/documentation/files/releasenotes new file mode 100644 index 0000000..c76d9d5 --- /dev/null +++ b/salt/profile/documentation/files/releasenotes @@ -0,0 +1,19 @@ +# Mapping file for release notes +# +# managed by salt - do not edit +# +# DISTVERSION|PRODUKT_NAME|PRODUKT_VERSION|RELEASE_NOTES_NAME|PRODUKT_ARCHS +# DISTVERSION : openSUSE version (from OBS download tree like: obs://openSUSE:13.1:Update/standard ) +# PRODUKT_NAME : Name of the product (will appear in tree) +# PRODUKT_VERSION : official Version number (12.3 or 13.1 - will appear in tree) +# RELEASE_NOTES_NAME : the name of the release-notes RPM +# PRODUKT_ARCHS : supported architectures (space separated) +# TARGET_REL_DIRS : target base directory where release notes should be unpacked (can/should stay empty) +# +# TODO: phase out factory and only provide Tumbleweed +obs://Documentation:Auto/openSUSE_Factory|openSUSE|factory|release-notes-openSUSE|x86_64| +obs://Documentation:Auto/openSUSE_Factory|openSUSE|Tumbleweed|release-notes-openSUSE|x86_64| +obs://Documentation:Auto/openSUSE_Leap_42.3|openSUSE|Leap/42.3|release-notes-openSUSE|x86_64| +obs://Documentation:Auto/openSUSE_Leap_15.0|openSUSE|Leap/15.0|release-notes-openSUSE|x86_64| +obs://Documentation:Auto/openSUSE_Leap_15.1|openSUSE|Leap/15.1|release-notes-openSUSE|x86_64| +obs://Documentation:Auto/openSUSE_Leap_15.2|openSUSE|Leap/15.2|release-notes-openSUSE|x86_64| diff --git a/salt/profile/documentation/files/update_release_notes b/salt/profile/documentation/files/update_release_notes new file mode 100644 index 0000000..40dd236 --- /dev/null +++ b/salt/profile/documentation/files/update_release_notes @@ -0,0 +1,301 @@ +#!/bin/bash +# +# managed by salt - do not edit +# +# Copyright (C) 2013-2014, SUSE Linux Products GmbH +# Copyright (C) 2015-2016, SUSE Linux GmbH +# Author: Lars Vogdt +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the Novell nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +BASEDIR="$HOME" +DESTDIR="/srv/www/vhosts/doc.opensuse.org/release-notes/" +RELEASENOTES_DATA="$BASEDIR/etc/releasenotes" +RELNOTESDIR="$BASEDIR/release-notes" +LOGFILE="$BASEDIR/var/log/$(basename $0).log" +DO_ECHO="" +VERBOSE="" +UNRPM='/usr/bin/unrpm' +UNPACK_TEMPDIR='/tmp' +TEMPDIR=$(mktemp -d /tmp/update_release_notes-XXXXXX) + +cleanup_and_exit() { + local exitcode="$1" + rm -rf "$TEMPDIR" + exit $exitcode +} + +usage() { + echo + echo "Usage: $(basename $0) [-t][-v]" + echo " -t : just test, do nothing" + echo " -v : be verbose" + echo + cleanup_and_exit $1 +} + +LOG(){ + LOG_DATE=`date "+%b %d %H:%M:%S"` + HOST=`hostname` + echo "$LOG_DATE $HOST : $1" >> $LOGFILE + if [ "$VERBOSE" = "yes" ]; then + echo "$1" + fi +} + +get_package_from_obs() { + local server="$1" + local localdir="$2" + local url=$(echo $3 | sed "s|:/|:|g") + local packagename="$4" + local archs="$5" + local attempt=$6 || 0 + attempt=$((attempt+1)) + if [ $attempt -gt 2 ]; then + echo "Could not find a valid RPM at ${server}/build/$url/$arch/_repository/${packagename} after $attempt attempts - giving up." >&2 + LOG "Could not find a valid RPM at ${server}/build/$url/$arch/_repository/${packagename} after $attempt attempts - giving up" + return + fi + for arch in $archs; do + $DO_ECHO mkdir -p $VERBOSE "$localdir/$arch" + $DO_ECHO curl -s -S $VERBOSE -o "$localdir/$arch"/${packagename}.rpm "${server}/build/$url/$arch/_repository/${packagename}" + FILETYPE=$(file --brief --mime "$localdir/$arch"/${packagename}.rpm) + FOUND='no' + case "$FILETYPE" in + *application/x-rpm*) + LOG "$localdir/$arch/${packagename}.rpm is a valid RPM - going forward" + FOUND='yes' + ;; + *text/plain*) + if [ "$VERBOSE" = "-v" ]; then + LOG "$localdir/$arch"/${packagename}.rpm has wrong mime - file content:" + cat "$localdir/$arch"/${packagename}.rpm" | tee "$LOGFILE" + fi + ;; + *) + LOG "$localdir/$arch/${packagename}.rpm has wrong/unknown mime type" + ;; + esac + if [ "$FOUND" == "no" ]; then + LOG "Did not found the wanted RPM at $url - trying different repos now" + case "$url" in + *:Update/standard) + url=${url/:Update/} + ;; + esac + get_package_from_obs "$server" "$localdir" "$url" "$packagename" "$arch" $attempt + fi + done +} + +output_package() { + PACKAGE="$1" + DESTINATION="$2" + case $DESTINATION in + "$RELNOTESDIR"/*) + ;; + *) + echo "ERROR: dest is $DESTINATION" >&2 + LOG "ERROR: dest is $DESTINATION" + return + ;; + esac + TMPDIR=$(mktemp -d $TEMPDIR/output_package-XXXXXX) + test -d "$TMPDIR" || return + pushd "$TMPDIR" 1>/dev/null + $DO_ECHO $UNRPM "$PACKAGE" 1>/dev/null 2>&1 + PACKNAME=$(rpm -qp --nosignature --nodigest --qf "%{NAME}.rpm" "$PACKAGE") + popd 1>/dev/null + $DO_ECHO rm -rf $VERBOSE "$DESTINATION" + $DO_ECHO mkdir -p "$DESTINATION" + $DO_ECHO cp -a $VERBOSE "$PACKAGE" "$DESTINATION/$PACKNAME" + $DO_ECHO cp -a $VERBOSE $TMPDIR/usr/share/doc/release-notes/*.{html,rtf,txt} "$DESTINATION"/ 2>/dev/null + $DO_ECHO cp -a $VERBOSE $TMPDIR/usr/share/doc/release-notes/*/*.{html,rtf,txt} "$DESTINATION"/ 2>/dev/null + for dir in $TMPDIR/usr/share/doc/release-notes/*/release-notes-static ; do + test -d "$dir" || continue + test -L "$dir/index.html" -a -f "$dir/index.html" || continue + $DO_ECHO cp -a $VERBOSE "$dir"/* "$DESTINATION"/ 2>/dev/null + N=`readlink "$DESTINATION/index.html"` + pushd "$DESTINATION" >/dev/null 2>/dev/null + $DO_ECHO rm $VERBOSE index.html + $DO_ECHO cp -f $VERBOSE "$N" index.html + popd "$DESTINATION" >/dev/null 2>/dev/null + done + for i in "$DESTINATION"/*.html ; do + test -L "$i" -a ! -f "$i" && $DO_ECHO rm $VERBOSE "$i" + done + pushd "$DESTINATION" 1>/dev/null + for i in RELEASE-NOTES.*.html; do + l="${i%.html}" + l="${l##*.}" + # use cp as symlinks do not work properly - see bnc #612554 for details (mod_proxy) + $DO_ECHO cp -f $VERBOSE "$i" index.html."$l" + done + if [ ! -e index.html.en ]; then + for i in en_US en_GB; do + if [ -e index.html.$i ]; then + $DO_ECHO mv -f $VERBOSE index.html.$i index.html.en + break + fi + done + fi + popd 1>/dev/null + if [ -d $TMPDIR/usr/share/doc/release-notes/*/static ]; then + $DO_ECHO cp -a $VERBOSE "$TMPDIR"/usr/share/doc/release-notes/*/static "$DESTINATION"/ + fi + rm -rf "$TMPDIR" +} + +create_index_file() { + DIRECTORY="$1" + case $DIRECTORY in + "$RELNOTESDIR"/*) + ;; + *) + echo "ERROR: create_index_file directory is $DIRECTORY" >&2 + LOG "ERROR: create_index_file directory is $DIRECTORY" + return + ;; + esac + test -d "$DIRECTORY" || return + if [ -f "$DIRECTORY/index.html" ]; then + LOG "WARNING: $DIRECTORY/index.html exists - skipping" + return + else + LOG "Creating $DIRECTORY/index.html" + fi + cat "$RELNOTESDIR/header.html.tmpl" > "$DIRECTORY/index.html" || return + echo "

Directory Index Of $(basename $DIRECTORY)/

" >> "$DIRECTORY/index.html" + echo "Parent Directory
" >> "$DIRECTORY/index.html" + echo "" >> "$DIRECTORY/index.html" + cat "$RELNOTESDIR/footer.html.tmpl" >> "$DIRECTORY/index.html" +} + +trap 'echo' SIGHUP SIGINT SIGQUIT +trap 'cleanup_and_exit 1' SIGTRAP SIGBUS SIGKILL SIGPIPE SIGTERM +trap 'cleanup_and_exit 0' EXIT + +while getopts 'htv' OPTION; do + case $OPTION in + h) usage 0 + ;; + t) DO_ECHO="echo " + ;; + v) VERBOSE="-v" + ;; + *) usage 1 + ;; + esac +done +shift $(( OPTIND - 1 )) + +if [ ! -x $UNRPM ]; then + echo "ERROR: $UNRPM not found or not executable" >&2 + LOG "ERROR: $UNRPM not found or not executable" + cleanup_and_exit 1 +fi + +if [ ! -f "$RELEASENOTES_DATA" ]; then + echo "ERROR: $RELEASENOTES_DATA not found" >&2 + LOG "ERROR: $RELEASENOTES_DATA not found" + cleanup_and_exit 1 +fi + +mkdir -p $(dirname $LOGFILE) || { echo "ERROR: could not create $(dirname $LOGFILE); LOG "ERROR: could not create $(dirname $LOGFILE); } + +while read line; do + ISBS="no" + if [[ "$line" == \#* ]]; then + continue + fi + if [[ "$line" == "" ]]; then + continue + fi + DISTVERSION=$(echo $line | cut -d "|" -f 1) + PRODUKT_NAME=$(echo $line | cut -d "|" -f 2) + PRODUKT_VERSION=$(echo $line | cut -d "|" -f 3) + RELEASENOTES_NAME=$(echo $line | cut -d "|" -f 4) + PRODUKT_ARCHS=$(echo $line | cut -d "|" -f 5) + case "$DISTVERSION" in + obs:*) + OBSDIR="$(mktemp -d "${TEMPDIR}/get_releasenotes-XXXXXX")" + DISTVERSION=$(echo ${DISTVERSION/obs:\/\//} | sed "s|:|:/|g") + DISTURL=$(echo ${DISTVERSION/obs:\/\//}) + ISBS="yes" + INSTANCE="https://build.opensuse.org/public" + get_package_from_obs "$INSTANCE" "${OBSDIR}/${DISTVERSION}" "$DISTURL" "$RELEASENOTES_NAME" "$PRODUKT_ARCHS" + ;; + *) + DISTVERSION="$DISTVERSION-" + ;; + esac + LOG "going for $OBSDIR/$DISTVERSION $PRODUKT_NAME/$PRODUKT_VERSION $RELEASENOTES_NAME" + $DO_ECHO mkdir -p "$RELNOTESDIR" || { echo "Could not create $RELNOTESDIR" >&2; cleanup_and_exit 1; } + if [ "$ISBS" == "yes" ]; then + # special handling for noarch packages + RELNOTES_RPM_IS_NOARCH=$(echo "$OBSDIR/$DISTVERSION/"*/$RELEASENOTES_NAME*.rpm | grep noarch) + for arch in $PRODUKT_ARCHS; do + if [ -n "$RELNOTES_RPM_IS_NOARCH" ]; then + repoarch="noarch" + else + repoarch="$arch" + fi + for relrpm in "$OBSDIR/$DISTVERSION/${repoarch}/"$RELEASENOTES_NAME*.rpm; do + test -f "$relrpm" || continue + RPM_NAME=$(rpm -qp --nosignature --nodigest --qf "%{NAME}" "$relrpm") + if [ "$RPM_NAME" = "$RELEASENOTES_NAME" ]; then + CHECKED=$(rpm --checksig "$relrpm" | grep -vE "md5 gpg OK|pgp md5 OK") + test -n "$CHECKED" && { echo "ERROR: $relrpm not signed" >&2 ; LOG "ERROR: $relrpm not signed"; continue ; } + PRODUKT_ARCH=$arch + output_package "$relrpm" "$RELNOTESDIR/$arch/$PRODUKT_NAME/$PRODUKT_VERSION" + # create_index_file "$RELNOTESDIR/$BUILDMACHINETYPE/$PRODUKT_NAME" + if [ "$RELEASENOTES_NAME" != "release-notes" ] ; then + $DO_ECHO cp -a "$RELNOTESDIR/$arch/$PRODUKT_NAME/$PRODUKT_VERSION"/$RELEASENOTES_NAME*.rpm "$RELNOTESDIR/$arch/$PRODUKT_NAME/$PRODUKT_VERSION/release-notes.rpm" + fi + else + continue + fi + done + done + date "+%Y-%m-%d %H:%M:%S" > "$RELNOTESDIR/.last_sync" + else + echo "Unknown DISTVERSION: $DISTVERSION - can currently handle \"obs://\" URLs only" >&2 + LOG "Unknown DISTVERSION: $DISTVERSION - can currently handle \"obs://\" URLs only" + cleanup_and_exit 1 + fi +done < "$RELEASENOTES_DATA" + +# Only after everything suceeds, we rsync the contents to /srv +rsync -arz --delete-after "$@" "$RELNOTESDIR/*" "$DESTDIR" diff --git a/salt/profile/documentation/init.sls b/salt/profile/documentation/init.sls new file mode 100644 index 0000000..affb084 --- /dev/null +++ b/salt/profile/documentation/init.sls @@ -0,0 +1,2 @@ +include: + - profile.documentation.release-notes diff --git a/salt/profile/documentation/release-notes.sls b/salt/profile/documentation/release-notes.sls new file mode 100644 index 0000000..7499d8f --- /dev/null +++ b/salt/profile/documentation/release-notes.sls @@ -0,0 +1,32 @@ +relsync_packages: + pkg.installed: + - pkgs: + - build # contains unrpm + +relsync: + user.present: + - home: /home/relsync + +{% set relsync_dirs = ['/home/relsync/bin', '/home/relsync/etc', '/srv/www/vhosts/doc.opensuse.org', '/srv/www/vhosts/doc.opensuse.org/release-notes'] %} + +{% for dir in relsync_dirs %} +{{ dir }}: + file.directory: + - user: relsync +{% endfor %} + +/home/relsync/etc/releasenotes: + file.managed: + - mode: 755 + - source: salt://profile/documentation/files/releasenotes + - user: relsync + +/home/relsync/bin/update_release_notes: + cron.present: + - user: relsync + - minute: 0 + - hour: */6 + file.managed: + - mode: 755 + - source: salt://profile/documentation/files/update_release_notes + - user: relsync diff --git a/salt/role/documentation.sls b/salt/role/documentation.sls new file mode 100644 index 0000000..2e1c086 --- /dev/null +++ b/salt/role/documentation.sls @@ -0,0 +1,3 @@ +include: + - profile.web.server.nginx + - profile.documentation