From df9c5347528f462754905117dfa60cbf90bad433 Mon Sep 17 00:00:00 2001 From: Michal Suchanek Date: Apr 21 2023 11:32:00 +0000 Subject: Merge branch 'packaging' into cve/linux-5.3 --- diff --git a/doc/README.SUSE b/doc/README.SUSE index fb737a0..5e9d6eb 100644 --- a/doc/README.SUSE +++ b/doc/README.SUSE @@ -136,7 +136,7 @@ recommended way to produce a binary kernel is: (6) Install the kernel and the modules (``make modules_install'', followed by ``make install''). This will automatically create - an initrd for the new kernel as well (see ``mkinitrd -h''). + an initrd for the new kernel as well (see ``dracut -h''). (7) Add the kernel to the boot manager. When using lilo, run ``lilo'' to update the boot map. diff --git a/rpm/constraints.in b/rpm/constraints.in index 712a0f8..dc46dfc 100644 --- a/rpm/constraints.in +++ b/rpm/constraints.in @@ -54,7 +54,7 @@ - + armv7l @@ -63,7 +63,7 @@ - 20 + 24 diff --git a/rpm/group-source-files.pl b/rpm/group-source-files.pl index 89fa8e8..3a8aa85 100755 --- a/rpm/group-source-files.pl +++ b/rpm/group-source-files.pl @@ -1,5 +1,6 @@ #!/usr/bin/perl +use File::Spec; use Getopt::Long; use strict; @@ -20,7 +21,12 @@ sub main sub scan { - my $loc = shift @_; + # Normalize file path, mainly to strip away the ending forward slash, + # or any double forward slashes. + my $loc = File::Spec->canonpath(shift @_); + # We cannot use an absolute path (e.g. /usr/src/linux-5.14.21-150500.41) + # during find because it's under build root, but rpm wants one later. + my $abs_loc = rpm_path($loc); my(@dev, @ndev); foreach $_ (`find "$loc"`) @@ -43,16 +49,12 @@ sub scan m{^\Q$loc\E/arch/[^/]+/tools\b} || m{^\Q$loc\E/include/[^/]+\b} || m{^\Q$loc\E/scripts\b}; - if (substr($_, 0, 1) ne "/") { - # We cannot use an absolute path during find, - # but rpm wants one later. - $_ = "/$_"; - } - $is_devel ? push(@dev, $_) : push(@ndev, $_); + my $abs_path = rpm_path($_); + $is_devel ? push(@dev, $abs_path) : push(@ndev, $abs_path); } - push(@dev, &calc_dirs("/$loc", \@dev)); - push(@ndev, &calc_dirs("/$loc", \@ndev)); + push(@dev, &calc_dirs($abs_loc, \@dev)); + push(@ndev, &calc_dirs($abs_loc, \@ndev)); return (\@dev, \@ndev); } @@ -62,11 +64,14 @@ sub calc_dirs my %dirs; foreach my $file (@$files) { - my $path = $file; + my ($volume,$path,$basename) = File::Spec->splitpath($file); + my @dirs = File::Spec->splitdir($path); do { - $path =~ s{/[^/]+$}{}; + # Always create $path from catdir() to avoid ending forward slash + $path = File::Spec->catdir(@dirs); $dirs{$path} = 1; - } while ($path ne $base and $path ne ""); + pop @dirs; + } while ($path ne $base); # This loop also makes sure that $base itself is included. } @@ -86,3 +91,11 @@ sub output print FH join("\n", @$ndev), "\n"; close FH; } + +sub rpm_path +{ + my $path = shift @_; + # Always prepend forward slash and let canonpath take care of + # duplicate forward slashes. + return File::Spec->canonpath("/$path"); +} diff --git a/rpm/kernel-binary.spec.in b/rpm/kernel-binary.spec.in index 31bd232..26351c2 100644 --- a/rpm/kernel-binary.spec.in +++ b/rpm/kernel-binary.spec.in @@ -20,7 +20,6 @@ %define srcversion @SRCVERSION@ %define patchversion @PATCHVERSION@ %define variant @VARIANT@%{nil} -%define vanilla_only @VANILLA_ONLY@ %define compress_modules @COMPRESS_MODULES@ %define compress_vmlinux @COMPRESS_VMLINUX@ %define livepatch @LIVEPATCH@%{nil} @@ -31,6 +30,7 @@ %define build_flavor @FLAVOR@ %define build_default ("%build_flavor" == "default") %define build_vanilla ("%build_flavor" == "vanilla") +%define vanilla_only %{lua: if (rpm.expand("%variant") == "-vanilla") then print(1) else print(0) end} %if ! %build_vanilla %define src_install_dir /usr/src/linux-%kernelrelease%variant @@ -173,6 +173,9 @@ Recommends: kernel-firmware # The following is copied to the -base subpackage as well # BEGIN COMMON DEPS Requires(pre): suse-kernel-rpm-scriptlets +Requires(post): suse-kernel-rpm-scriptlets +Requires: suse-kernel-rpm-scriptlets +Requires(preun): suse-kernel-rpm-scriptlets Requires(postun): suse-kernel-rpm-scriptlets Requires(pre): coreutils awk # For /usr/lib/module-init-tools/weak-modules2 @@ -184,21 +187,16 @@ Requires(post): modutils # test -x update-bootloader, having perl-Bootloader is not a hard requirement. # But, there is no way to tell rpm or yast to schedule the installation # of perl-Bootloader before kernel-binary.rpm if both are in the list of -# packages to install/update. Likewise, this is true for mkinitrd. +# packages to install/update. Likewise, this is true for dracut. # Need a perl-Bootloader with /usr/lib/bootloader/bootloader_entry Requires(post): perl-Bootloader >= 0.4.15 -%if %build_vanilla -Requires(post): mkinitrd -%else -# Require a mkinitrd that can handle usbhid/hid-generic built-in (bnc#773559) -Requires(post): mkinitrd >= 2.7.1 -%endif +Requires(post): dracut # Install the package providing /etc/SuSE-release early enough, so that # the grub entry has correct title (bnc#757565) Requires(post): distribution-release -# Do not install p-b and mkinitrd for the install check, the %post script is +# Do not install p-b and dracut for the install check, the %post script is # able to handle this -#!BuildIgnore: perl-Bootloader mkinitrd distribution-release +#!BuildIgnore: perl-Bootloader dracut distribution-release # Remove some packages that are installed automatically by the build system, # but are not needed to build the kernel #!BuildIgnore: autoconf automake gettext-runtime libtool cvs gettext-tools udev insserv @@ -393,7 +391,7 @@ awk '{ cd linux-%srcversion %_sourcedir/apply-patches \ -%if %{build_vanilla} +%if %{build_vanilla} && ! %vanilla_only --vanilla \ %endif %_sourcedir/series.conf .. $SYMBOLS @@ -1172,7 +1170,7 @@ Requires: %{name}_%_target_cpu = %version-%source_rel Requires(pre): coreutils awk Requires(post): modutils Requires(post): perl-Bootloader -Requires(post): mkinitrd +Requires(post): dracut @PROVIDES_OBSOLETES_EXTRA@ %obsolete_rebuilds %name-extra Supplements: packageand(product(SLED):%{name}_%_target_cpu) @@ -1238,7 +1236,7 @@ Requires: %name-extra_%_target_cpu = %version-%source_rel Requires(pre): coreutils awk Requires(post): modutils Requires(post): perl-Bootloader -Requires(post): mkinitrd +Requires(post): dracut @PROVIDES_OBSOLETES_OPTIONAL@ %obsolete_rebuilds %name-optional Supplements: packageand(product(Leap):%{name}_%_target_cpu) @@ -1327,7 +1325,7 @@ Summary: Development files necessary for building kernel modules Group: Development/Sources Provides: %name-devel = %version-%source_rel Provides: multiversion(kernel) -%if ! %build_vanilla +%if ! %build_vanilla && ! %vanilla_only Requires: kernel-devel%variant = %version-%source_rel Recommends: make Recommends: gcc diff --git a/rpm/kernel-cert-subpackage b/rpm/kernel-cert-subpackage index 04a71b3..ed475d7 100644 --- a/rpm/kernel-cert-subpackage +++ b/rpm/kernel-cert-subpackage @@ -2,6 +2,9 @@ Summary: UEFI Secure Boot Certificate For Package %{-n*}-kmp Group: System/Kernel Requires(pre): suse-kernel-rpm-scriptlets +Requires(post): suse-kernel-rpm-scriptlets +Requires: suse-kernel-rpm-scriptlets +Requires(preun): suse-kernel-rpm-scriptlets Requires(postun): suse-kernel-rpm-scriptlets %description -n %{-n*}-ueficert This package contains the UEFI Secure Boot certificate used to sign diff --git a/rpm/kernel-module-subpackage b/rpm/kernel-module-subpackage index 3a3d18c..749ed17 100644 --- a/rpm/kernel-module-subpackage +++ b/rpm/kernel-module-subpackage @@ -1,5 +1,5 @@ %package -n %{-n*}-kmp-%1 -%define _this_kmp_kernel_version k%(echo %2 | sed -r 'y/-/_/; s/^(2\.6\.[0-9]+)_/\\1.0_/; # use 2.6.x.0 for mainline kernels') +%define _this_kmp_kernel_version k%(echo %2 | sed -r 'y/-/_/') %define _this_kmp_version %{-v*}_%_this_kmp_kernel_version Version: %_this_kmp_version Release: %{-r*} @@ -27,26 +27,14 @@ Provides: multiversion(kernel) Provides: %{-n*}-kmp-%1-%_this_kmp_kernel_version Requires: coreutils grep Requires(pre): suse-kernel-rpm-scriptlets +Requires(post): suse-kernel-rpm-scriptlets +Requires: suse-kernel-rpm-scriptlets +Requires(preun): suse-kernel-rpm-scriptlets Requires(postun): suse-kernel-rpm-scriptlets %{-c:Requires: %{-n*}-ueficert} Enhances: kernel-%1 Supplements: packageand(kernel-%1:%{-n*}) Conflicts: %{-n*}-kmp-%1-%_this_kmp_kernel_version -%if "%1" == "default" -Obsoletes: %{-n*}-kmp-trace -%ifarch %ix86 -Obsoletes: %{-n*}-kmp-vmi -%endif -%ifarch x86_64 -Obsoletes: %{-n*}-kmp-desktop -%endif -%ifarch %ix86 x86_64 -Obsoletes: %{-n*}-kmp-xen -%endif -%endif -%if "%1" == "pae" -Obsoletes: %{-n*}-kmp-desktop -%endif AutoReqProv: on %define run_if_exists run_if_exists() { \ diff --git a/rpm/kernel-obs-build.spec.in b/rpm/kernel-obs-build.spec.in index c5999c8..10c8859 100644 --- a/rpm/kernel-obs-build.spec.in +++ b/rpm/kernel-obs-build.spec.in @@ -21,7 +21,6 @@ %define patchversion @PATCHVERSION@ %define variant @VARIANT@%{nil} -%define vanilla_only @VANILLA_ONLY@ %include %_sourcedir/kernel-spec-macros @@ -31,8 +30,8 @@ BuildRequires: device-mapper BuildRequires: util-linux %if 0%{?suse_version} -%if %vanilla_only -%define kernel_flavor -vanilla +%if "@OBS_BUILD_VARIANT@" +%define kernel_flavor @OBS_BUILD_VARIANT@ %else %ifarch %ix86 %define kernel_flavor -pae @@ -53,14 +52,7 @@ BuildRequires: kernel %endif ExclusiveArch: @ARCHS@ -%if 0%{?suse_version} < 1315 -# For SLE 11 -BuildRequires: mkinitrd -BuildRequires: perl-Bootloader -BuildRoot: %{_tmppath}/%{name}-%{version}-build -%else BuildRequires: dracut -%endif Summary: package kernel and initrd for OBS VM builds License: GPL-2.0-only Group: SLES @@ -145,12 +137,6 @@ ROOT="" %define kernel_name Image %endif -%if 0%{?suse_version} && 0%{?suse_version} < 1315 -# For SLE 11 -/sbin/mkinitrd $ROOT \ - -m "$KERNEL_MODULES" \ - -k /boot/%{kernel_name}-*-default -M /boot/System.map-*-default -i /tmp/initrd.kvm -B -%else # --host-only mode is needed for unlimited TasksMax workaround (boo#965564) dracut --reproducible --host-only --no-hostonly-cmdline \ --no-early-microcode --nofscks --strip --hardlink \ @@ -162,7 +148,6 @@ dracut --reproducible --host-only --no-hostonly-cmdline \ --compress "zstd -19 -T0" \ %endif $(echo /boot/%{kernel_name}-*%{kernel_flavor} | sed -n -e 's,[^-]*-\(.*'%{kernel_flavor}'\),\1,p') -%endif #cleanup rm -rf /usr/lib/dracut/modules.d/80obs diff --git a/rpm/kernel-source.spec.in b/rpm/kernel-source.spec.in index 0754cb7..0f74e3d 100644 --- a/rpm/kernel-source.spec.in +++ b/rpm/kernel-source.spec.in @@ -19,7 +19,6 @@ %define srcversion @SRCVERSION@ %define patchversion @PATCHVERSION@ %define variant @VARIANT@%{nil} -%define vanilla_only @VANILLA_ONLY@ %include %_sourcedir/kernel-spec-macros @@ -231,11 +230,7 @@ sed -ie 's,/lib/modules/,%{kernel_module_directory}/,' linux-%kernelrelease%vari %endif %if %do_vanilla -%if %vanilla_only - mv \ -%else cp -al \ -%endif linux-%kernelrelease%variant linux-%kernelrelease-vanilla cd linux-%kernelrelease-vanilla %_sourcedir/apply-patches --vanilla %_sourcedir/series.conf %my_builddir %symbols @@ -245,7 +240,6 @@ rm -f $(find . -name ".gitignore") cd .. %endif -%if ! %vanilla_only cd linux-%kernelrelease%variant %_sourcedir/apply-patches %_sourcedir/series.conf %my_builddir %symbols rm -f $(find . -name ".gitignore") @@ -256,10 +250,8 @@ fi # Hardlink duplicate files automatically (from package fdupes). %fdupes $PWD cd .. -%endif popd -%if ! %vanilla_only # Install the documentation and example Kernel Module Package. DOC=/usr/share/doc/packages/%name-%kernelrelease mkdir -p %buildroot/$DOC @@ -286,7 +278,6 @@ perl "%_sourcedir/group-source-files.pl" \ -D "$OLDPWD/devel.files" -N "$OLDPWD/nondevel.files" \ -L "%src_install_dir" popd -%endif find %{buildroot}/usr/src/linux* -type f -name '*.[ch]' -perm /0111 -exec chmod -v a-x {} + # OBS checks don't like /usr/bin/env in script interpreter lines @@ -301,7 +292,6 @@ done ts="$(head -n1 %_sourcedir/source-timestamp)" find %buildroot/usr/src/linux* ! -type l | xargs touch -d "$ts" -%if ! %vanilla_only %post %relink_function @@ -329,7 +319,6 @@ relink linux-%kernelrelease%variant /usr/src/linux%variant /usr/lib/rpm/kernel/* %endif -%endif %if %do_vanilla diff --git a/rpm/mkspec b/rpm/mkspec index c415073..0e21c34 100755 --- a/rpm/mkspec +++ b/rpm/mkspec @@ -35,8 +35,9 @@ my @kmps = read_kmps(); # config.sh variables my %vars = parse_config_sh(); -my ($srcversion, $variant, $vanilla_only) = - ($vars{'SRCVERSION'}, $vars{'VARIANT'}, $vars{'VANILLA_ONLY'}); +my ($srcversion, $variant, $obs_build_variant) = + ($vars{'SRCVERSION'}, $vars{'VARIANT'}, $vars{'OBS_BUILD_VARIANT'}); +$obs_build_variant = ($obs_build_variant ? $variant : "" ); my $compress_modules = 'none'; my $compress_vmlinux = 'gz'; my $build_dtbs = (); @@ -60,7 +61,6 @@ if (defined($vars{'LIVEPATCH_RT'})) { $livepatch_rt = $vars{'LIVEPATCH_RT'}; $livepatch_rt = "" if $livepatch_rt =~ /^(0+|no|none)$/i; } -$vanilla_only ||= "0"; if (!defined ($rpmrelease)) { $rpmrelease = $vars{'RELEASE'} || 0; } @@ -114,7 +114,7 @@ my $commit_full = get_commit(1); my %macros = ( VARIANT => $variant, - VANILLA_ONLY => $vanilla_only, + OBS_BUILD_VARIANT => $obs_build_variant . "%{nil}", SRCVERSION => $srcversion, PATCHVERSION => $patchversion, RPMVERSION => $rpmversion, @@ -216,14 +216,16 @@ if ($variant eq "") { } # kernel-obs-*.spec -if (!$variant) { +if (!$variant || $obs_build_variant) { my @default_archs; - - if ($vanilla_only) { - @default_archs = arch2rpm(@{$flavor_archs{vanilla}}); + my $flavor = $obs_build_variant; + if ($flavor) { + $flavor =~ s/^-//; } else { - @default_archs = arch2rpm(@{$flavor_archs{default}}); + $flavor = 'default'; } + + @default_archs = arch2rpm(@{$flavor_archs{$flavor}}); # No kernel-obs-* for 32bit ppc and x86 @default_archs = grep { $_ ne "ppc" && $_ ne '%ix86' } @default_archs; my $default_archs = join(" ", @default_archs); @@ -234,7 +236,7 @@ if (!$variant) { } # dtb-*.spec -if (!$variant && $build_dtbs) { +if ((!$variant || $obs_build_variant) && $build_dtbs) { do_spec('dtb', "dtb.spec.in", %macros); print "./mkspec-dtb $all_archs\n"; system("./mkspec-dtb $all_archs\n"); diff --git a/scripts/bs-upload-kernel b/scripts/bs-upload-kernel index c97c25a..6456d25 100755 --- a/scripts/bs-upload-kernel +++ b/scripts/bs-upload-kernel @@ -108,7 +108,7 @@ if (-e "$dir/klp-symbols") { } my @remove_packages = qw(kernel-dummy); if (!$enable_checks) { - push(@remove_packages, "post-build-checks", "rpmlint-Factory", + push(@remove_packages, "rpmlint-Factory", "post-build-checks-malwarescan"); } my $prjconf = ""; diff --git a/scripts/bugzilla-cli b/scripts/bugzilla-cli index c1eca50..f122f47 100755 --- a/scripts/bugzilla-cli +++ b/scripts/bugzilla-cli @@ -12,7 +12,7 @@ if scriptdir[0] != '/': sys.path.insert(0, scriptdir) from bugzilla import _cli -if _cli.DEFAULT_BZ != "https://apibugzilla.suse.com/xmlrpc.cgi": +if _cli.DEFAULT_BZ != "https://apibugzilla.suse.com": raise RuntimeError("Use of this script requires the SUSE version of python-bugzilla.") _cli.main() diff --git a/scripts/bugzilla-create b/scripts/bugzilla-create index 5d6de3c..c643f39 100755 --- a/scripts/bugzilla-create +++ b/scripts/bugzilla-create @@ -1,6 +1,5 @@ #!/bin/bash -URL="https://apibugzilla.suse.com/xmlrpc.cgi" COMPONENT="Kernel" COMMENT="This is an automated report for a proactive fix, documented below." @@ -176,7 +175,7 @@ if [ ! -e "${DIR}/bugzilla-cli" ]; then exit 1 fi -BUGZILLA="${DIR}/bugzilla-cli --bugzilla ${URL} --ensure-logged-in" +BUGZILLA="${DIR}/bugzilla-cli --ensure-logged-in" cleanup () { rm -rf ${tmpdir} diff --git a/scripts/bugzilla-resolve b/scripts/bugzilla-resolve index 7fd6237..f2d1fd6 100755 --- a/scripts/bugzilla-resolve +++ b/scripts/bugzilla-resolve @@ -1,7 +1,5 @@ #!/bin/bash -URL="https://apibugzilla.suse.com/xmlrpc.cgi" - resolve_one() { ${BUGZILLA} modify -l "Automated update: This patch was committed to the kernel git repository. Closing as FIXED." -k FIXED $1 } @@ -12,7 +10,7 @@ if [ ! -e "${DIR}/bugzilla-cli" ]; then exit 1 fi -BUGZILLA="${DIR}/bugzilla-cli --bugzilla ${URL} --ensure-logged-in" +BUGZILLA="${DIR}/bugzilla-cli --ensure-logged-in" if [ "$#" -eq 0 ]; then echo "usage: $(basename $0) [bug ids ...]" >&2 diff --git a/scripts/bugzilla/__init__.py b/scripts/bugzilla/__init__.py index 74f5514..95a52cd 100644 --- a/scripts/bugzilla/__init__.py +++ b/scripts/bugzilla/__init__.py @@ -3,19 +3,15 @@ # Copyright (C) 2007, 2008 Red Hat Inc. # Author: Will Woods # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. from .apiversion import version, __version__ from .base import Bugzilla -from .transport import BugzillaError -from .rhbugzilla import RHBugzilla +from .exceptions import BugzillaError from .oldclasses import (Bugzilla3, Bugzilla32, Bugzilla34, Bugzilla36, Bugzilla4, Bugzilla42, Bugzilla44, - NovellBugzilla, RHBugzilla3, RHBugzilla4) + NovellBugzilla, RHBugzilla, RHBugzilla3, RHBugzilla4) # This is the public API. If you are explicitly instantiating any other diff --git a/scripts/bugzilla/_authfiles.py b/scripts/bugzilla/_authfiles.py new file mode 100644 index 0000000..bdb977e --- /dev/null +++ b/scripts/bugzilla/_authfiles.py @@ -0,0 +1,179 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +import configparser +import os +from logging import getLogger +import urllib.parse + +from ._util import listify + +log = getLogger(__name__) + + +def _parse_hostname(url): + # If http://example.com is passed, netloc=example.com path="" + # If just example.com is passed, netloc="" path=example.com + parsedbits = urllib.parse.urlparse(url) + return parsedbits.netloc or parsedbits.path + + +def _makedirs(path): + if os.path.exists(os.path.dirname(path)): + return + os.makedirs(os.path.dirname(path), 0o700) + + +def _default_cache_location(filename): + """ + Determine default location for passed xdg filename. + example: ~/.cache/python-bugzilla/bugzillarc + """ + return os.path.expanduser("~/.cache/python-bugzilla/%s" % filename) + + +class _BugzillaRCFile(object): + """ + Helper class for interacting with bugzillarc files + """ + @staticmethod + def get_default_configpaths(): + paths = [ + '/etc/bugzillarc', + '~/.bugzillarc', + '~/.config/python-bugzilla/bugzillarc', + ] + return paths + + def __init__(self): + self._cfg = None + self._configpaths = None + self.set_configpaths(None) + + def set_configpaths(self, configpaths): + configpaths = [os.path.expanduser(p) for p in + listify(configpaths or [])] + + cfg = configparser.ConfigParser() + read_files = cfg.read(configpaths) + if read_files: + log.info("Found bugzillarc files: %s", read_files) + + self._cfg = cfg + self._configpaths = configpaths or [] + + def get_configpaths(self): + return self._configpaths[:] + + def get_default_url(self): + """ + Grab a default URL from bugzillarc [DEFAULT] url=X + """ + cfgurl = self._cfg.defaults().get("url", None) + if cfgurl is not None: + log.debug("bugzillarc: found cli url=%s", cfgurl) + return cfgurl + + def parse(self, url): + """ + Find the section for the passed URL domain, and return all the fields + """ + section = "" + log.debug("bugzillarc: Searching for config section matching %s", url) + + urlhost = _parse_hostname(url) + for sectionhost in sorted(self._cfg.sections()): + # If the section is just a hostname, make it match + # If the section has a / in it, do a substring match + if "/" not in sectionhost: + if sectionhost == urlhost: + section = sectionhost + elif sectionhost in url: + section = sectionhost + if section: + log.debug("bugzillarc: Found matching section: %s", section) + break + + if not section: + log.debug("bugzillarc: No section found") + return {} + return dict(self._cfg.items(section)) + + + def save_api_key(self, url, api_key): + """ + Save the API_KEY in the config file. We use the last file + in the configpaths list, which is the one with the highest + precedence. + """ + configpaths = self.get_configpaths() + if not configpaths: + return None + + config_filename = configpaths[-1] + section = _parse_hostname(url) + cfg = configparser.ConfigParser() + cfg.read(config_filename) + + if section not in cfg.sections(): + cfg.add_section(section) + + cfg.set(section, 'api_key', api_key.strip()) + + _makedirs(config_filename) + with open(config_filename, 'w') as configfile: + cfg.write(configfile) + + return config_filename + + +class _BugzillaTokenCache(object): + """ + Class for interacting with a .bugzillatoken cache file + """ + @staticmethod + def get_default_path(): + return _default_cache_location("bugzillatoken") + + def __init__(self): + self._filename = None + self._cfg = None + + def _get_domain(self, url): + domain = urllib.parse.urlparse(url)[1] + if domain and domain not in self._cfg.sections(): + self._cfg.add_section(domain) + return domain + + def get_value(self, url): + domain = self._get_domain(url) + if domain and self._cfg.has_option(domain, 'token'): + return self._cfg.get(domain, 'token') + return None + + def set_value(self, url, value): + if self.get_value(url) == value: + return + + domain = self._get_domain(url) + if value is None: + self._cfg.remove_option(domain, 'token') + else: + self._cfg.set(domain, 'token', value) + + if self._filename: + _makedirs(self._filename) + with open(self._filename, 'w') as _cfg: + log.debug("Saving to _cfg") + self._cfg.write(_cfg) + + def get_filename(self): + return self._filename + + def set_filename(self, filename): + log.debug("Using tokenfile=%s", filename) + cfg = configparser.ConfigParser() + if filename: + cfg.read(filename) + self._filename = filename + self._cfg = cfg diff --git a/scripts/bugzilla/_backendbase.py b/scripts/bugzilla/_backendbase.py new file mode 100644 index 0000000..b81e108 --- /dev/null +++ b/scripts/bugzilla/_backendbase.py @@ -0,0 +1,288 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +from logging import getLogger + +import requests + +log = getLogger(__name__) + + +class _BackendBase(object): + """ + Backends are thin wrappers around the different bugzilla API paradigms + (XMLRPC, REST). This base class defines the public API for the rest of + the code, but this is all internal to the library. + """ + def __init__(self, url, bugzillasession): + self._url = url + self._bugzillasession = bugzillasession + + + @staticmethod + def probe(url): + try: + requests.head(url).raise_for_status() + return True # pragma: no cover + except Exception as e: + log.debug("Failed to probe url=%s : %s", url, str(e)) + return False + + + ################# + # Internal APIs # + ################# + + def get_xmlrpc_proxy(self): + """ + Provides the raw XMLRPC proxy to API users of Bugzilla._proxy + """ + raise NotImplementedError() + + def is_rest(self): + """ + :returns: True if this is the REST backend + """ + return False + + def is_xmlrpc(self): + """ + :returns: True if this is the XMLRPC backend + """ + return False + + + ###################### + # Bugzilla info APIs # + ###################### + + def bugzilla_version(self): + """ + Fetch bugzilla version string + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bugzilla.html#version + """ + raise NotImplementedError() + + + ####################### + # Bug attachment APIs # + ####################### + + def bug_attachment_get(self, attachment_ids, paramdict): + """ + Fetch bug attachments IDs. One part of: + http://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#get-attachment + """ + raise NotImplementedError() + + def bug_attachment_get_all(self, bug_ids, paramdict): + """ + Fetch all bug attachments IDs. One part of + http://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#get-attachment + """ + raise NotImplementedError() + + def bug_attachment_create(self, bug_ids, data, paramdict): + """ + Create a bug attachment + http://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#create-attachment + + :param data: raw Bytes data of the attachment to attach. API will + encode this correctly if you pass it in and 'data' is not in + paramdict. + """ + raise NotImplementedError() + + def bug_attachment_update(self, attachment_ids, paramdict): + """ + Update a bug attachment + http://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#update-attachment + """ + raise NotImplementedError() + + + ############ + # bug APIs # + ############ + + def bug_comments(self, bug_ids, paramdict): + """ + Fetch bug comments + http://bugzilla.readthedocs.io/en/latest/api/core/v1/comment.html#get-comments + """ + raise NotImplementedError() + + def bug_create(self, paramdict): + """ + Create a new bug + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#create-bug + """ + raise NotImplementedError() + + def bug_fields(self, paramdict): + """ + Query available bug field values + http://bugzilla.readthedocs.io/en/latest/api/core/v1/field.html#fields + """ + raise NotImplementedError() + + def bug_get(self, bug_ids, aliases, paramdict): + """ + Lookup bug data by ID + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#get-bug + """ + raise NotImplementedError() + + def bug_history(self, bug_ids, paramdict): + """ + Lookup bug history + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#bug-history + """ + raise NotImplementedError() + + def bug_search(self, paramdict): + """ + Search/query bugs + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#search-bugs + """ + raise NotImplementedError() + + def bug_update(self, bug_ids, paramdict): + """ + Update bugs + http://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#update-bug + """ + raise NotImplementedError() + + def bug_update_tags(self, bug_ids, paramdict): + """ + Update bug tags + https://www.bugzilla.org/docs/4.4/en/html/api/Bugzilla/WebService/Bug.html#update_tags + """ + raise NotImplementedError() + + + ################## + # Component APIs # + ################## + + def component_create(self, paramdict): + """ + Create component + https://bugzilla.readthedocs.io/en/latest/api/core/v1/component.html#create-component + """ + raise NotImplementedError() + + def component_update(self, paramdict): + """ + Update component + https://bugzilla.readthedocs.io/en/latest/api/core/v1/component.html#update-component + """ + raise NotImplementedError() + + + ############################### + # ExternalBugs extension APIs # + ############################### + + def externalbugs_add(self, paramdict): + """ + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#add-external-bug + """ + raise NotImplementedError() + + def externalbugs_update(self, paramdict): + """ + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#update-external-bug + """ + raise NotImplementedError() + + def externalbugs_remove(self, paramdict): + """ + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#remove-external-bug + """ + raise NotImplementedError() + + + ############## + # Group APIs # + ############## + + def group_get(self, paramdict): + """ + https://bugzilla.readthedocs.io/en/latest/api/core/v1/group.html#get-group + """ + raise NotImplementedError() + + + ################ + # Product APIs # + ################ + + def product_get(self, paramdict): + """ + Fetch product details + http://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html#get-product + """ + raise NotImplementedError() + + def product_get_accessible(self): + """ + List accessible products + http://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html#list-products + """ + raise NotImplementedError() + + def product_get_enterable(self): + """ + List enterable products + http://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html#list-products + """ + raise NotImplementedError() + + def product_get_selectable(self): + """ + List selectable products + http://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html#list-products + """ + raise NotImplementedError() + + + ############# + # User APIs # + ############# + + def user_create(self, paramdict): + """ + Create user + http://bugzilla.readthedocs.io/en/latest/api/core/v1/user.html#create-user + """ + raise NotImplementedError() + + def user_get(self, paramdict): + """ + Get user info + http://bugzilla.readthedocs.io/en/latest/api/core/v1/user.html#get-user + """ + raise NotImplementedError() + + def user_login(self, paramdict): + """ + Log in to bugzilla + http://bugzilla.readthedocs.io/en/latest/api/core/v1/user.html#login + """ + raise NotImplementedError() + + def user_logout(self): + """ + Log out of bugzilla + http://bugzilla.readthedocs.io/en/latest/api/core/v1/user.html#logout + """ + raise NotImplementedError() + + def user_update(self, paramdict): + """ + Update user + http://bugzilla.readthedocs.io/en/latest/api/core/v1/user.html#update-user + """ + raise NotImplementedError() diff --git a/scripts/bugzilla/_backendrest.py b/scripts/bugzilla/_backendrest.py new file mode 100644 index 0000000..3abe49c --- /dev/null +++ b/scripts/bugzilla/_backendrest.py @@ -0,0 +1,193 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +import base64 +import json +import logging +import os + +from ._backendbase import _BackendBase +from .exceptions import BugzillaError +from ._util import listify + + +log = logging.getLogger(__name__) + + +def _update_key(indict, updict, key): + if key not in indict: + indict[key] = {} + indict[key].update(updict.get(key, {})) + + +class _BackendREST(_BackendBase): + """ + Internal interface for direct calls to bugzilla's REST API + """ + def __init__(self, url, bugzillasession): + _BackendBase.__init__(self, url, bugzillasession) + self._bugzillasession.set_rest_defaults() + + + ######################### + # Internal REST helpers # + ######################### + + def _handle_response(self, text): + try: + ret = dict(json.loads(text)) + except Exception: # pragma: no cover + log.debug("Failed to parse REST response. Output is:\n%s", text) + raise + + if ret.get("error", False): + raise BugzillaError(ret["message"], code=ret["code"]) + return ret + + def _op(self, method, apiurl, paramdict=None): + fullurl = os.path.join(self._url, apiurl.lstrip("/")) + log.debug("Bugzilla REST %s %s params=%s", method, fullurl, paramdict) + + data = None + authparams = self._bugzillasession.get_auth_params() + if method == "GET": + authparams.update(paramdict or {}) + else: + data = json.dumps(paramdict or {}) + + response = self._bugzillasession.request(method, fullurl, data=data, + params=authparams) + return self._handle_response(response.text) + + def _get(self, *args, **kwargs): + return self._op("GET", *args, **kwargs) + def _put(self, *args, **kwargs): + return self._op("PUT", *args, **kwargs) + def _post(self, *args, **kwargs): + return self._op("POST", *args, **kwargs) + + + ####################### + # API implementations # + ####################### + + def get_xmlrpc_proxy(self): + raise BugzillaError("You are using the bugzilla REST API, " + "so raw XMLRPC access is not provided.") + def is_rest(self): + return True + + def bugzilla_version(self): + return self._get("/version") + + def bug_create(self, paramdict): + return self._post("/bug", paramdict) + def bug_fields(self, paramdict): + return self._get("/field/bug", paramdict) + def bug_get(self, bug_ids, aliases, paramdict): + data = paramdict.copy() + data["id"] = listify(bug_ids) + data["alias"] = listify(aliases) + ret = self._get("/bug", data) + return ret + + def bug_attachment_get(self, attachment_ids, paramdict): + # XMLRPC supported mutiple fetch at once, but not REST + ret = {} + for attid in listify(attachment_ids): + out = self._get("/bug/attachment/%s" % attid, paramdict) + _update_key(ret, out, "attachments") + _update_key(ret, out, "bugs") + return ret + + def bug_attachment_get_all(self, bug_ids, paramdict): + # XMLRPC supported mutiple fetch at once, but not REST + ret = {} + for bugid in listify(bug_ids): + out = self._get("/bug/%s/attachment" % bugid, paramdict) + _update_key(ret, out, "attachments") + _update_key(ret, out, "bugs") + return ret + + def bug_attachment_create(self, bug_ids, data, paramdict): + if data is not None and "data" not in paramdict: + paramdict["data"] = base64.b64encode(data).decode("utf-8") + paramdict["ids"] = listify(bug_ids) + return self._post("/bug/%s/attachment" % paramdict["ids"][0], + paramdict) + + def bug_attachment_update(self, attachment_ids, paramdict): + paramdict["ids"] = listify(attachment_ids) + return self._put("/bug/attachment/%s" % paramdict["ids"][0], paramdict) + + def bug_comments(self, bug_ids, paramdict): + # XMLRPC supported mutiple fetch at once, but not REST + ret = {} + for bugid in bug_ids: + out = self._get("/bug/%s/comment" % bugid, paramdict) + _update_key(ret, out, "bugs") + return ret + def bug_history(self, bug_ids, paramdict): + # XMLRPC supported mutiple fetch at once, but not REST + ret = {"bugs": []} + for bugid in bug_ids: + out = self._get("/bug/%s/history" % bugid, paramdict) + ret["bugs"].extend(out.get("bugs", [])) + return ret + + def bug_search(self, paramdict): + return self._get("/bug", paramdict) + def bug_update(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._put("/bug/%s" % data["ids"][0], data) + def bug_update_tags(self, bug_ids, paramdict): + raise BugzillaError("No REST API available for bug_update_tags") + + def component_create(self, paramdict): + return self._post("/component", paramdict) + def component_update(self, paramdict): + if "ids" in paramdict: + apiurl = str(listify(paramdict["ids"])[0]) # pragma: no cover + if "names" in paramdict: + apiurl = ("%(product)s/%(component)s" % + listify(paramdict["names"])[0]) + return self._put("/component/%s" % apiurl, paramdict) + + def externalbugs_add(self, paramdict): # pragma: no cover + raise BugzillaError( + "No REST API available yet for externalbugs_add") + def externalbugs_remove(self, paramdict): # pragma: no cover + raise BugzillaError( + "No REST API available yet for externalbugs_remove") + def externalbugs_update(self, paramdict): # pragma: no cover + raise BugzillaError( + "No REST API available yet for externalbugs_update") + + def group_get(self, paramdict): + return self._get("/group", paramdict) + + def product_get(self, paramdict): + return self._get("/product/get", paramdict) + def product_get_accessible(self): + return self._get("/product_accessible") + def product_get_enterable(self): + return self._get("/product_enterable") + def product_get_selectable(self): + return self._get("/product_selectable") + + def user_create(self, paramdict): + return self._post("/user", paramdict) + def user_get(self, paramdict): + return self._get("/user", paramdict) + def user_login(self, paramdict): + return self._get("/login", paramdict) + def user_logout(self): + return self._get("/logout") + def user_update(self, paramdict): + urlid = None + if "ids" in paramdict: + urlid = listify(paramdict["ids"])[0] # pragma: no cover + if "names" in paramdict: + urlid = listify(paramdict["names"])[0] + return self._put("/user/%s" % urlid, paramdict) diff --git a/scripts/bugzilla/_backendxmlrpc.py b/scripts/bugzilla/_backendxmlrpc.py new file mode 100644 index 0000000..0558b35 --- /dev/null +++ b/scripts/bugzilla/_backendxmlrpc.py @@ -0,0 +1,228 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +from logging import getLogger +import sys +from xmlrpc.client import (Binary, Fault, ProtocolError, + ServerProxy, Transport) + +from requests import RequestException + +from ._backendbase import _BackendBase +from .exceptions import BugzillaError +from ._util import listify + + +log = getLogger(__name__) + + +class _BugzillaXMLRPCTransport(Transport): + def __init__(self, bugzillasession): + if hasattr(Transport, "__init__"): + Transport.__init__(self, use_datetime=False) + + self.__bugzillasession = bugzillasession + self.__bugzillasession.set_xmlrpc_defaults() + self.__seen_valid_xml = False + + # Override Transport.user_agent + self.user_agent = self.__bugzillasession.get_user_agent() + + + ############################ + # Bugzilla private helpers # + ############################ + + def __request_helper(self, url, request_body): + """ + A helper method to assist in making a request and parsing the response. + """ + response = None + # pylint: disable=try-except-raise + # pylint: disable=raise-missing-from + try: + response = self.__bugzillasession.request( + "POST", url, data=request_body) + + return self.parse_response(response) + except RequestException as e: + if not response: + raise + raise ProtocolError( # pragma: no cover + url, response.status_code, str(e), response.headers) + except Fault: + raise + except Exception: + msg = str(sys.exc_info()[1]) + if not self.__seen_valid_xml: + msg += "\nThe URL may not be an XMLRPC URL: %s" % url + e = BugzillaError(msg) + # pylint: disable=attribute-defined-outside-init + e.__traceback__ = sys.exc_info()[2] + # pylint: enable=attribute-defined-outside-init + raise e + + + ###################### + # Tranport overrides # + ###################### + + def parse_response(self, response): + """ + Override Transport.parse_response + """ + parser, unmarshaller = self.getparser() + msg = response.text.encode('utf-8') + try: + parser.feed(msg) + except Exception: # pragma: no cover + log.debug("Failed to parse this XMLRPC response:\n%s", msg) + raise + + self.__seen_valid_xml = True + parser.close() + return unmarshaller.close() + + def request(self, host, handler, request_body, verbose=0): + """ + Override Transport.request + """ + # Setting self.verbose here matches overrided request() behavior + # pylint: disable=attribute-defined-outside-init + self.verbose = verbose + + url = "%s://%s%s" % (self.__bugzillasession.get_scheme(), + host, handler) + + # xmlrpclib fails to escape \r + request_body = request_body.replace(b'\r', b' ') + + return self.__request_helper(url, request_body) + + +class _BugzillaXMLRPCProxy(ServerProxy, object): + """ + Override of xmlrpc ServerProxy, to insert bugzilla API auth + into the XMLRPC request data + """ + def __init__(self, uri, bugzillasession, *args, **kwargs): + self.__bugzillasession = bugzillasession + transport = _BugzillaXMLRPCTransport(self.__bugzillasession) + ServerProxy.__init__(self, uri, transport, *args, **kwargs) + + def _ServerProxy__request(self, methodname, params): + """ + Overrides ServerProxy _request method + """ + # params is a singleton tuple, enforced by xmlrpc.client.dumps + newparams = params and params[0].copy() or {} + + log.debug("XMLRPC call: %s(%s)", methodname, newparams) + authparams = self.__bugzillasession.get_auth_params() + authparams.update(newparams) + + # pylint: disable=no-member + ret = ServerProxy._ServerProxy__request( + self, methodname, (authparams,)) + # pylint: enable=no-member + + return ret + + +class _BackendXMLRPC(_BackendBase): + """ + Internal interface for direct calls to bugzilla's XMLRPC API + """ + def __init__(self, url, bugzillasession): + _BackendBase.__init__(self, url, bugzillasession) + self._xmlrpc_proxy = _BugzillaXMLRPCProxy(url, self._bugzillasession) + + def get_xmlrpc_proxy(self): + return self._xmlrpc_proxy + def is_xmlrpc(self): + return True + + def bugzilla_version(self): + return self._xmlrpc_proxy.Bugzilla.version() + + def bug_attachment_get(self, attachment_ids, paramdict): + data = paramdict.copy() + data["attachment_ids"] = listify(attachment_ids) + return self._xmlrpc_proxy.Bug.attachments(data) + def bug_attachment_get_all(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._xmlrpc_proxy.Bug.attachments(data) + def bug_attachment_create(self, bug_ids, data, paramdict): + pdata = paramdict.copy() + pdata["ids"] = listify(bug_ids) + if data is not None and "data" not in paramdict: + pdata["data"] = Binary(data) + return self._xmlrpc_proxy.Bug.add_attachment(pdata) + def bug_attachment_update(self, attachment_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(attachment_ids) + return self._xmlrpc_proxy.Bug.update_attachment(data) + + def bug_comments(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._xmlrpc_proxy.Bug.comments(data) + def bug_create(self, paramdict): + return self._xmlrpc_proxy.Bug.create(paramdict) + def bug_fields(self, paramdict): + return self._xmlrpc_proxy.Bug.fields(paramdict) + def bug_get(self, bug_ids, aliases, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) or [] + data["ids"] += listify(aliases) or [] + return self._xmlrpc_proxy.Bug.get(data) + def bug_history(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._xmlrpc_proxy.Bug.history(data) + def bug_search(self, paramdict): + return self._xmlrpc_proxy.Bug.search(paramdict) + def bug_update(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._xmlrpc_proxy.Bug.update(data) + def bug_update_tags(self, bug_ids, paramdict): + data = paramdict.copy() + data["ids"] = listify(bug_ids) + return self._xmlrpc_proxy.Bug.update_tags(data) + + def component_create(self, paramdict): + return self._xmlrpc_proxy.Component.create(paramdict) + def component_update(self, paramdict): + return self._xmlrpc_proxy.Component.update(paramdict) + + def externalbugs_add(self, paramdict): + return self._xmlrpc_proxy.ExternalBugs.add_external_bug(paramdict) + def externalbugs_update(self, paramdict): + return self._xmlrpc_proxy.ExternalBugs.update_external_bug(paramdict) + def externalbugs_remove(self, paramdict): + return self._xmlrpc_proxy.ExternalBugs.remove_external_bug(paramdict) + + def group_get(self, paramdict): + return self._xmlrpc_proxy.Group.get(paramdict) + + def product_get(self, paramdict): + return self._xmlrpc_proxy.Product.get(paramdict) + def product_get_accessible(self): + return self._xmlrpc_proxy.Product.get_accessible_products() + def product_get_enterable(self): + return self._xmlrpc_proxy.Product.get_enterable_products() + def product_get_selectable(self): + return self._xmlrpc_proxy.Product.get_selectable_products() + + def user_create(self, paramdict): + return self._xmlrpc_proxy.User.create(paramdict) + def user_get(self, paramdict): + return self._xmlrpc_proxy.User.get(paramdict) + def user_login(self, paramdict): + return self._xmlrpc_proxy.User.login(paramdict) + def user_logout(self): + return self._xmlrpc_proxy.User.logout() + def user_update(self, paramdict): + return self._xmlrpc_proxy.User.update(paramdict) diff --git a/scripts/bugzilla/_cli.py b/scripts/bugzilla/_cli.py index 1c10586..e0b4924 100644 --- a/scripts/bugzilla/_cli.py +++ b/scripts/bugzilla/_cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # bugzilla - a commandline frontend for the python bugzilla module # @@ -6,39 +6,30 @@ # Author: Will Woods # Author: Cole Robinson # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - -from __future__ import print_function +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. +import argparse +import base64 +import datetime +import errno +import json import locale from logging import getLogger, DEBUG, INFO, WARN, StreamHandler, Formatter -import argparse import os import re import socket import sys import tempfile - -# pylint: disable=import-error -if sys.version_info[0] >= 3: - # pylint: disable=no-name-in-module,redefined-builtin - from xmlrpc.client import Fault, ProtocolError - from urllib.parse import urlparse - basestring = (str, bytes) -else: - from xmlrpclib import Fault, ProtocolError - from urlparse import urlparse -# pylint: enable=import-error +import urllib.parse +import xmlrpc.client import requests.exceptions import bugzilla -DEFAULT_BZ = 'https://apibugzilla.suse.com/xmlrpc.cgi' + +DEFAULT_BZ = 'https://apibugzilla.suse.com' format_field_re = re.compile("%{([a-z0-9_]+)(?::([^}]*))?}") @@ -49,33 +40,15 @@ log = getLogger(bugzilla.__name__) # Util helpers # ################ -def _is_unittest(): - return bool(os.getenv("__BUGZILLA_UNITTEST")) - - def _is_unittest_debug(): return bool(os.getenv("__BUGZILLA_UNITTEST_DEBUG")) -def to_encoding(ustring): - string = '' - if isinstance(ustring, basestring): - string = ustring - elif ustring is not None: - string = str(ustring) - - if sys.version_info[0] >= 3: - return string - - preferred = locale.getpreferredencoding() - if _is_unittest(): - preferred = "UTF-8" - return string.encode(preferred, 'replace') - - def open_without_clobber(name, *args): - '''Try to open the given file with the given mode; if that filename exists, - try "name.1", "name.2", etc. until we find an unused filename.''' + """ + Try to open the given file with the given mode; if that filename exists, + try "name.1", "name.2", etc. until we find an unused filename. + """ fd = None count = 1 orig_name = name @@ -83,31 +56,17 @@ def open_without_clobber(name, *args): try: fd = os.open(name, os.O_CREAT | os.O_EXCL, 0o666) except OSError as err: - if err.errno == os.errno.EEXIST: + if err.errno == errno.EEXIST: name = "%s.%i" % (orig_name, count) count += 1 - else: - raise IOError(err.errno, err.strerror, err.filename) + else: # pragma: no cover + raise IOError(err.errno, err.strerror, err.filename) from None fobj = open(name, *args) if fd != fobj.fileno(): os.close(fd) return fobj -def get_default_url(): - """ - Grab a default URL from bugzillarc [DEFAULT] url=X - """ - from bugzilla.base import _open_bugzillarc - cfg = _open_bugzillarc() - if cfg: - cfgurl = cfg.defaults().get("url", None) - if cfgurl is not None: - log.debug("bugzillarc: found cli url=%s", cfgurl) - return cfgurl - return DEFAULT_BZ - - def setup_logging(debug, verbose): handler = StreamHandler(sys.stderr) handler.setFormatter(Formatter( @@ -123,7 +82,7 @@ def setup_logging(debug, verbose): log.setLevel(WARN) if _is_unittest_debug(): - log.setLevel(DEBUG) + log.setLevel(DEBUG) # pragma: no cover ################## @@ -134,11 +93,13 @@ def _setup_root_parser(): epilog = 'Try "bugzilla COMMAND --help" for command-specific help.' p = argparse.ArgumentParser(epilog=epilog) - default_url = get_default_url() + default_url = bugzilla.Bugzilla.get_rcfile_default_url() + if not default_url: + default_url = DEFAULT_BZ # General bugzilla connection options p.add_argument('--bugzilla', default=default_url, - help="bugzilla XMLRPC URI. default: %s" % default_url) + help="bugzilla URI. default: %s" % default_url) p.add_argument("--nosslverify", dest="sslverify", action="store_false", default=True, help="Don't error on invalid bugzilla SSL certificate") @@ -150,6 +111,9 @@ def _setup_root_parser(): 'specified command.') p.add_argument('--username', help="Log in with this username") p.add_argument('--password', help="Log in with this password") + p.add_argument('--restrict-login', action="store_true", + help="The session (login token) will be restricted to " + "the current IP address.") p.add_argument('--ensure-logged-in', action="store_true", help="Raise an error if we aren't logged in to bugzilla. " @@ -161,8 +125,7 @@ def _setup_root_parser(): help="Don't save any bugzilla cookies or tokens to disk, and " "don't use any pre-existing credentials.") - p.add_argument('--cookiefile', default=None, - help="cookie file to use for bugzilla authentication") + p.add_argument('--cookiefile', default=None, help=argparse.SUPPRESS) p.add_argument('--tokenfile', default=None, help="token file to use for bugzilla authentication") @@ -195,8 +158,26 @@ def _parser_add_output_options(p): outg.add_argument('--oneline', action='store_const', dest='output', const='oneline', help="one line summary of the bug (useful for scripts)") + outg.add_argument('--json', action='store_const', dest='output', + const='json', help="output contents in json format") + outg.add_argument("--includefield", action="append", + help="Pass the field name to bugzilla include_fields list. " + "Only the fields passed to include_fields are returned " + "by the bugzilla server. " + "This can be specified multiple times.") + outg.add_argument("--extrafield", action="append", + help="Pass the field name to bugzilla extra_fields list. " + "When used with --json this can be used to request " + "bugzilla to return values for non-default fields. " + "This can be specified multiple times.") + outg.add_argument("--excludefield", action="append", + help="Pass the field name to bugzilla exclude_fields list. " + "When used with --json this can be used to request " + "bugzilla to not return values for a field. " + "This can be specified multiple times.") outg.add_argument('--raw', action='store_const', dest='output', - const='raw', help="raw output of the bugzilla contents") + const='raw', help="raw output of the bugzilla contents. This " + "format is unstable and difficult to parse. Use --json instead.") outg.add_argument('--outputformat', help="Print output in the form given. " "You can use RPM-style tags that match bug " @@ -250,6 +231,10 @@ def _parser_add_bz_fields(rootp, command): p.add_argument('--cc', action="append", help="CC list") p.add_argument('-a', '--assigned_to', '--assignee', help="Bug assignee") p.add_argument('-q', '--qa_contact', help='QA contact') + if cmd_modify: + p.add_argument("--minor-update", action="store_true", + help="Request bugzilla to not send any " + "email about this change") if not cmd_new: p.add_argument('-f', '--flag', action='append', @@ -274,15 +259,11 @@ def _parser_add_bz_fields(rootp, command): # Put this at the end, so it sticks out more p.add_argument('--field', metavar="FIELD=VALUE", action="append", dest="fields", - help="Manually specify a bugzilla XMLRPC field. FIELD is " - "the raw name used by the bugzilla instance. For example if your " + help="Manually specify a bugzilla API field. FIELD is " + "the raw name used by the bugzilla instance. For example, if your " "bugzilla instance has a custom field cf_my_field, do:\n" " --field cf_my_field=VALUE") - # Used by unit tests, not for end user consumption - p.add_argument('--__test-return-result', action="store_true", - dest="test_return_result", help=argparse.SUPPRESS) - if not cmd_modify: _parser_add_output_options(rootp) @@ -294,11 +275,14 @@ def _setup_action_new_parser(subparsers): "Options that take multiple values accept comma separated lists, " "including --cc, --blocks, --dependson, --groups, and --keywords.") p = subparsers.add_parser("new", description=description) - - _parser_add_bz_fields(p, "new") p.add_argument('--no-refresh', action='store_true', help='Do not refresh bug after creating') + _parser_add_bz_fields(p, "new") + g = p.add_argument_group("'new' specific options") + g.add_argument('--private', action='store_true', default=False, + help='Mark new comment as private') + def _setup_action_query_parser(subparsers): description = ("List bug reports that match the given criteria. " @@ -344,10 +328,6 @@ def _setup_action_query_parser(subparsers): help=argparse.SUPPRESS) p.add_argument('-W', '--status_whiteboard_type', help=argparse.SUPPRESS) - p.add_argument('-B', '--booleantype', - help=argparse.SUPPRESS) - p.add_argument('--boolean_query', action="append", - help=argparse.SUPPRESS) p.add_argument('--fixed_in_type', help=argparse.SUPPRESS) @@ -399,7 +379,7 @@ def _setup_action_modify_parser(subparsers): def _setup_action_attach_parser(subparsers): usage = """ bugzilla attach --file=FILE --desc=DESC [--type=TYPE] BUGID [BUGID...] -bugzilla attach --get=ATTACHID --getall=BUGID [...] +bugzilla attach --get=ATTACHID --getall=BUGID [--ignore-obsolete] [...] bugzilla attach --type=TYPE BUGID [BUGID...]""" description = "Attach files or download attachments." p = subparsers.add_parser("attach", description=description, usage=usage) @@ -416,17 +396,29 @@ bugzilla attach --type=TYPE BUGID [BUGID...]""" default=[], help="Download the attachment with the given ID") p.add_argument("--getall", "--get-all", metavar="BUGID", action="append", default=[], help="Download all attachments on the given bug") - p.add_argument('-l', '--comment', '--long_desc', help="Add comment with attachment") + p.add_argument('--ignore-obsolete', action="store_true", + help='Do not download attachments marked as obsolete.') + p.add_argument('-l', '--comment', '--long_desc', + help="Add comment with attachment") + p.add_argument('--private', action='store_true', default=False, + help='Mark new comment as private') def _setup_action_login_parser(subparsers): - usage = 'bugzilla login [username [password]]' - description = "Log into bugzilla and save a login cookie or token." + usage = 'bugzilla login [--api-key] [username [password]]' + description = """Log into bugzilla and save a login cookie or token. +Note: These tokens are short-lived, and future Bugzilla versions will no +longer support token authentication at all. Please use a +~/.config/python-bugzilla/bugzillarc file with an API key instead, or +use 'bugzilla login --api-key' and we will save it for you.""" p = subparsers.add_parser("login", description=description, usage=usage) - p.add_argument("pos_username", nargs="?", help="Optional username", - metavar="username") - p.add_argument("pos_password", nargs="?", help="Optional password", - metavar="password") + p.add_argument('--api-key', action='store_true', default=False, + help='Prompt for and save an API key into bugzillarc, ' + 'rather than prompt for username and password.') + p.add_argument("pos_username", nargs="?", help="Optional username ", + metavar="username") + p.add_argument("pos_password", nargs="?", help="Optional password ", + metavar="password") def setup_parser(): @@ -446,12 +438,9 @@ def setup_parser(): # Command routines # #################### -def _merge_field_opts(query, opt, parser): +def _merge_field_opts(query, fields, parser): # Add any custom fields if specified - if opt.fields is None: - return - - for f in opt.fields: + for f in fields: try: f, v = f.split('=', 1) query[f] = v @@ -494,7 +483,7 @@ def _do_query(bz, opt, parser): # Alias for EndOfLife bug statuses stat = ['VERIFIED', 'RELEASE_PENDING', 'RESOLVED'] elif val == 'OPEN': - # non-Closed statuses + # non-RESOLVED statuses stat = ['NEW', 'ASSIGNED', 'MODIFIED', 'ON_DEV', 'ON_QA', 'VERIFIED', 'RELEASE_PENDING', 'POST'] opt.status = stat @@ -512,7 +501,7 @@ def _do_query(bz, opt, parser): setattr(opt, optname, val.split(",")) include_fields = None - if opt.output == 'raw': + if opt.output in ['raw', 'json']: # 'raw' always does a getbug() call anyways, so just ask for ID back include_fields = ['id'] @@ -537,55 +526,89 @@ def _do_query(bz, opt, parser): if include_fields is not None: include_fields.sort() - built_query = bz.build_query( - product=opt.product or None, - component=opt.component or None, - sub_component=opt.sub_component or None, - version=opt.version or None, - reporter=opt.reporter or None, - bug_id=opt.id or None, - short_desc=opt.summary or None, - long_desc=opt.comment or None, - cc=opt.cc or None, - assigned_to=opt.assigned_to or None, - qa_contact=opt.qa_contact or None, - status=opt.status or None, - blocked=opt.blocked or None, - dependson=opt.dependson or None, - keywords=opt.keywords or None, - keywords_type=opt.keywords_type or None, - url=opt.url or None, - url_type=opt.url_type or None, - status_whiteboard=opt.whiteboard or None, - status_whiteboard_type=opt.status_whiteboard_type or None, - fixed_in=opt.fixed_in or None, - fixed_in_type=opt.fixed_in_type or None, - flag=opt.flag or None, - alias=opt.alias or None, - qa_whiteboard=opt.qa_whiteboard or None, - devel_whiteboard=opt.devel_whiteboard or None, - boolean_query=opt.boolean_query or None, - bug_severity=opt.severity or None, - priority=opt.priority or None, - target_release=opt.target_release or None, - target_milestone=opt.target_milestone or None, - emailtype=opt.emailtype or None, - booleantype=opt.booleantype or None, - include_fields=include_fields, - quicksearch=opt.quicksearch or None, - savedsearch=opt.savedsearch or None, - savedsearch_sharer_id=opt.savedsearch_sharer_id or None, - tags=opt.tags or None) - - _merge_field_opts(built_query, opt, parser) + kwopts = {} + if opt.product: + kwopts["product"] = opt.product + if opt.component: + kwopts["component"] = opt.component + if opt.sub_component: + kwopts["sub_component"] = opt.sub_component + if opt.version: + kwopts["version"] = opt.version + if opt.reporter: + kwopts["reporter"] = opt.reporter + if opt.id: + kwopts["bug_id"] = opt.id + if opt.summary: + kwopts["short_desc"] = opt.summary + if opt.comment: + kwopts["long_desc"] = opt.comment + if opt.cc: + kwopts["cc"] = opt.cc + if opt.assigned_to: + kwopts["assigned_to"] = opt.assigned_to + if opt.qa_contact: + kwopts["qa_contact"] = opt.qa_contact + if opt.status: + kwopts["status"] = opt.status + if opt.blocked: + kwopts["blocked"] = opt.blocked + if opt.dependson: + kwopts["dependson"] = opt.dependson + if opt.keywords: + kwopts["keywords"] = opt.keywords + if opt.keywords_type: + kwopts["keywords_type"] = opt.keywords_type + if opt.url: + kwopts["url"] = opt.url + if opt.url_type: + kwopts["url_type"] = opt.url_type + if opt.whiteboard: + kwopts["status_whiteboard"] = opt.whiteboard + if opt.status_whiteboard_type: + kwopts["status_whiteboard_type"] = opt.status_whiteboard_type + if opt.fixed_in: + kwopts["fixed_in"] = opt.fixed_in + if opt.fixed_in_type: + kwopts["fixed_in_type"] = opt.fixed_in_type + if opt.flag: + kwopts["flag"] = opt.flag + if opt.alias: + kwopts["alias"] = opt.alias + if opt.qa_whiteboard: + kwopts["qa_whiteboard"] = opt.qa_whiteboard + if opt.devel_whiteboard: + kwopts["devel_whiteboard"] = opt.devel_whiteboard + if opt.severity: + kwopts["bug_severity"] = opt.severity + if opt.priority: + kwopts["priority"] = opt.priority + if opt.target_release: + kwopts["target_release"] = opt.target_release + if opt.target_milestone: + kwopts["target_milestone"] = opt.target_milestone + if opt.emailtype: + kwopts["emailtype"] = opt.emailtype + if include_fields: + kwopts["include_fields"] = include_fields + if opt.quicksearch: + kwopts["quicksearch"] = opt.quicksearch + if opt.savedsearch: + kwopts["savedsearch"] = opt.savedsearch + if opt.savedsearch_sharer_id: + kwopts["savedsearch_sharer_id"] = opt.savedsearch_sharer_id + if opt.tags: + kwopts["tags"] = opt.tags + + built_query = bz.build_query(**kwopts) + if opt.fields: + _merge_field_opts(built_query, opt.fields, parser) built_query.update(q) q = built_query - if not q: + if not q: # pragma: no cover parser.error("'query' command requires additional arguments") - if opt.test_return_result: - return q return bz.query(q) @@ -603,18 +626,18 @@ def _do_info(bz, opt): return ret productname = (opt.components or opt.component_owners or opt.versions) - include_fields = ["name", "id"] fastcomponents = (opt.components and not opt.active_components) + + include_fields = ["name", "id"] + if opt.components or opt.component_owners: + include_fields += ["components.name"] + if opt.component_owners: + include_fields += ["components.default_assigned_to"] + if opt.active_components: + include_fields += ["components.is_active"] + if opt.versions: include_fields += ["versions"] - if opt.component_owners: - include_fields += [ - "components.default_assigned_to", - "components.name", - ] - if (opt.active_components and - any(["components" in i for i in include_fields])): - include_fields += ["components.is_active"] bz.refresh_products(names=productname and [productname] or None, include_fields=include_fields) @@ -635,14 +658,12 @@ def _do_info(bz, opt): elif opt.versions: proddict = bz.getproducts()[0] for v in proddict['versions']: - if v["is_active"]: - print(to_encoding(v["name"])) + print(str(v["name"] or '')) elif opt.component_owners: details = bz.getcomponentsdetails(productname) for c in sorted(_filter_components(details)): - print(to_encoding(u"%s: %s" % (c, - details[c]['default_assigned_to']))) + print("%s: %s" % (c, details[c]['default_assigned_to'])) def _convert_to_outputformat(output): @@ -673,92 +694,139 @@ def _convert_to_outputformat(output): fmt += "#%{bug_id} %{status} %{assigned_to} %{component}\t" fmt += "[%{target_milestone}] %{flags} %{cve}" - else: + else: # pragma: no cover raise RuntimeError("Unknown output type '%s'" % output) return fmt -def _format_output(bz, opt, buglist): - if opt.output == 'raw': - buglist = bz.getbugs([b.bug_id for b in buglist]) - for b in buglist: - print("Bugzilla %s: " % b.bug_id) - for attrname in sorted(b.__dict__): - print(to_encoding(u"ATTRIBUTE[%s]: %s" % - (attrname, b.__dict__[attrname]))) - print("\n\n") - return +def _xmlrpc_converter(obj): + if "DateTime" in str(obj.__class__): + # xmlrpc DateTime object. Convert to date format that + # bugzilla REST API outputs + dobj = datetime.datetime.strptime(str(obj), '%Y%m%dT%H:%M:%S') + return dobj.isoformat() + "Z" + if "Binary" in str(obj.__class__): + # xmlrpc Binary object. Convert to base64 + return base64.b64encode(obj.data).decode("utf-8") + raise RuntimeError( + "Unexpected JSON conversion class=%s" % obj.__class__) + - def bug_field(matchobj): - # whiteboard and flag allow doing - # %{whiteboard:devel} and %{flag:needinfo} - # That's what 'rest' matches - (fieldname, rest) = matchobj.groups() - - if fieldname == "whiteboard" and rest: - fieldname = rest + "_" + fieldname - - if fieldname == "flag" and rest: - val = b.get_flag_status(rest) - - elif fieldname == "flags" or fieldname == "flags_requestee": - tmpstr = [] - for f in getattr(b, "flags", []): - requestee = f.get('requestee', "") - if fieldname == "flags": - requestee = "" - if fieldname == "flags_requestee": - if requestee == "": - continue - tmpstr.append("%s" % requestee) - else: - tmpstr.append("%s%s%s" % - (f['name'], f['status'], requestee)) - - val = ",".join(tmpstr) - - elif fieldname == "cve": - cves = [] - for key in getattr(b, "keywords", []): - # grab CVE from keywords and blockers - if key.find("Security") == -1: +def _format_output_json(buglist): + out = {"bugs": [b.get_raw_data() for b in buglist]} + s = json.dumps(out, default=_xmlrpc_converter, indent=2, sort_keys=True) + print(s) + + +def _format_output_raw(buglist): + for b in buglist: + print("Bugzilla %s: " % b.bug_id) + SKIP_NAMES = ["bugzilla"] + for attrname in sorted(b.__dict__): + if attrname in SKIP_NAMES: + continue + if attrname.startswith("_"): + continue + print("ATTRIBUTE[%s]: %s" % (attrname, b.__dict__[attrname])) + print("\n\n") + + +def _bug_field_repl_cb(bz, b, matchobj): + # whiteboard and flag allow doing + # %{whiteboard:devel} and %{flag:needinfo} + # That's what 'rest' matches + (fieldname, rest) = matchobj.groups() + + if fieldname == "whiteboard" and rest: + fieldname = rest + "_" + fieldname + + if fieldname == "flag" and rest: + val = b.get_flag_status(rest) + + elif fieldname in ["flags", "flags_requestee"]: + tmpstr = [] + for f in getattr(b, "flags", []): + requestee = f.get('requestee', "") + if fieldname == "flags": + requestee = "" + if fieldname == "flags_requestee": + if requestee == "": continue - for bl in b.blocks: - cvebug = bz.getbug(bl) - for cb in cvebug.alias: - if cb.find("CVE") == -1: - continue - if cb.strip() not in cves: - cves.append(cb) - val = ",".join(cves) - - elif fieldname == "comments": - val = "" - for c in getattr(b, "comments", []): - val += ("\n* %s - %s:\n%s\n" % (c['time'], - c.get("creator", c.get("author", "")), c['text'])) - - elif fieldname == "external_bugs": - val = "" - for e in getattr(b, "external_bugs", []): - url = e["type"]["full_url"].replace("%id%", e["ext_bz_bug_id"]) - if not val: - val += "\n" - val += "External bug: %s\n" % url - - elif fieldname == "__unicode__": - val = b.__unicode__() - else: - val = getattr(b, fieldname, "") + tmpstr.append("%s" % requestee) + else: + tmpstr.append("%s%s%s" % + (f['name'], f['status'], requestee)) + + val = ",".join(tmpstr) + + elif fieldname == "cve": + cves = [] + for key in getattr(b, "keywords", []): + # grab CVE from keywords and blockers + if key.find("Security") == -1: + continue + for bl in b.blocks: + cvebug = bz.getbug(bl) + for cb in cvebug.alias: + if (cb.find("CVE") != -1 and + cb.strip() not in cves): + cves.append(cb) + val = ",".join(cves) + + elif fieldname == "comments": + val = "" + for c in getattr(b, "comments", []): + val += ("\n* %s - %s:\n%s\n" % (c['time'], + c.get("creator", c.get("author", "")), c['text'])) + + elif fieldname == "external_bugs": + val = "" + for e in getattr(b, "external_bugs", []): + url = e["type"]["full_url"].replace("%id%", e["ext_bz_bug_id"]) + if not val: + val += "\n" + val += "External bug: %s\n" % url + + elif fieldname == "__unicode__": + val = b.__unicode__() + else: + val = getattr(b, fieldname, "") + + vallist = isinstance(val, list) and val or [val] + val = ','.join([str(v or '') for v in vallist]) - vallist = isinstance(val, list) and val or [val] - val = ','.join([to_encoding(v) for v in vallist]) + return val - return val + +def _format_output(bz, opt, buglist): + if opt.output in ['raw', 'json']: + include_fields = None + exclude_fields = None + extra_fields = None + + if opt.includefield: + include_fields = opt.includefield + if opt.excludefield: + exclude_fields = opt.excludefield + if opt.extrafield: + extra_fields = opt.extrafield + + buglist = bz.getbugs([b.bug_id for b in buglist], + include_fields=include_fields, + exclude_fields=exclude_fields, + extra_fields=extra_fields) + if opt.output == 'json': + _format_output_json(buglist) + if opt.output == 'raw': + _format_output_raw(buglist) + return for b in buglist: - print(format_field_re.sub(bug_field, opt.outputformat)) + # pylint: disable=cell-var-from-loop + def cb(matchobj): + return _bug_field_repl_cb(bz, b, matchobj) + print(format_field_re.sub(cb, opt.outputformat)) def _parse_triset(vallist, checkplus=True, checkminus=True, checkequal=True, @@ -796,33 +864,53 @@ def _do_new(bz, opt, parser): return _parse_triset(val, checkplus=False, checkminus=False, checkequal=False, splitcomma=True)[0] - ret = bz.build_createbug( - blocks=parse_multi(opt.blocked) or None, - cc=parse_multi(opt.cc) or None, - component=opt.component or None, - depends_on=parse_multi(opt.dependson) or None, - description=opt.comment or None, - groups=parse_multi(opt.groups) or None, - keywords=parse_multi(opt.keywords) or None, - op_sys=opt.os or None, - platform=opt.arch or None, - priority=opt.priority or None, - product=opt.product or None, - severity=opt.severity or None, - summary=opt.summary or None, - url=opt.url or None, - version=opt.version or None, - assigned_to=opt.assigned_to or None, - qa_contact=opt.qa_contact or None, - sub_component=opt.sub_component or None, - alias=opt.alias or None, - comment_tags=opt.comment_tag or None, - ) - - _merge_field_opts(ret, opt, parser) - - if opt.test_return_result: - return ret + kwopts = {} + if opt.blocked: + kwopts["blocks"] = parse_multi(opt.blocked) + if opt.cc: + kwopts["cc"] = parse_multi(opt.cc) + if opt.component: + kwopts["component"] = opt.component + if opt.dependson: + kwopts["depends_on"] = parse_multi(opt.dependson) + if opt.comment: + kwopts["description"] = opt.comment + if opt.groups: + kwopts["groups"] = parse_multi(opt.groups) + if opt.keywords: + kwopts["keywords"] = parse_multi(opt.keywords) + if opt.os: + kwopts["op_sys"] = opt.os + if opt.arch: + kwopts["platform"] = opt.arch + if opt.priority: + kwopts["priority"] = opt.priority + if opt.product: + kwopts["product"] = opt.product + if opt.severity: + kwopts["severity"] = opt.severity + if opt.summary: + kwopts["summary"] = opt.summary + if opt.url: + kwopts["url"] = opt.url + if opt.version: + kwopts["version"] = opt.version + if opt.assigned_to: + kwopts["assigned_to"] = opt.assigned_to + if opt.qa_contact: + kwopts["qa_contact"] = opt.qa_contact + if opt.sub_component: + kwopts["sub_component"] = opt.sub_component + if opt.alias: + kwopts["alias"] = opt.alias + if opt.comment_tag: + kwopts["comment_tags"] = opt.comment_tag + if opt.private: + kwopts["comment_private"] = opt.private + + ret = bz.build_createbug(**kwopts) + if opt.fields: + _merge_field_opts(ret, opt.fields, parser) b = bz.createbug(ret) if not opt.no_refresh: @@ -861,50 +949,96 @@ def _do_modify(bz, parser, opt): for f in opt.flag: flags.append({"name": f[:-1], "status": f[-1]}) - update = bz.build_update( - assigned_to=opt.assigned_to or None, - comment=opt.comment or None, - comment_private=opt.private or None, - component=opt.component or None, - product=opt.product or None, - blocks_add=add_blk or None, - blocks_remove=rm_blk or None, - blocks_set=set_blk, - url=opt.url or None, - cc_add=add_cc or None, - cc_remove=rm_cc or None, - depends_on_add=add_deps or None, - depends_on_remove=rm_deps or None, - depends_on_set=set_deps, - groups_add=add_groups or None, - groups_remove=rm_groups or None, - keywords_add=add_key or None, - keywords_remove=rm_key or None, - keywords_set=set_key, - op_sys=opt.os or None, - platform=opt.arch or None, - priority=opt.priority or None, - qa_contact=opt.qa_contact or None, - severity=opt.severity or None, - status=status, - summary=opt.summary or None, - version=opt.version or None, - reset_assigned_to=opt.reset_assignee or None, - reset_qa_contact=opt.reset_qa_contact or None, - resolution=opt.close or None, - target_release=opt.target_release or None, - target_milestone=opt.target_milestone or None, - dupe_of=opt.dupeid or None, - fixed_in=opt.fixed_in or None, - whiteboard=set_wb and set_wb[0] or None, - devel_whiteboard=set_devwb and set_devwb[0] or None, - internal_whiteboard=set_intwb and set_intwb[0] or None, - qa_whiteboard=set_qawb and set_qawb[0] or None, - sub_component=opt.sub_component or None, - alias=opt.alias or None, - flags=flags or None, - comment_tags=opt.comment_tag or None, - ) + update_opts = {} + + if opt.assigned_to: + update_opts["assigned_to"] = opt.assigned_to + if opt.comment: + update_opts["comment"] = opt.comment + if opt.private: + update_opts["comment_private"] = opt.private + if opt.component: + update_opts["component"] = opt.component + if opt.product: + update_opts["product"] = opt.product + if add_blk: + update_opts["blocks_add"] = add_blk + if rm_blk: + update_opts["blocks_remove"] = rm_blk + if set_blk is not None: + update_opts["blocks_set"] = set_blk + if opt.url: + update_opts["url"] = opt.url + if add_cc: + update_opts["cc_add"] = add_cc + if rm_cc: + update_opts["cc_remove"] = rm_cc + if add_deps: + update_opts["depends_on_add"] = add_deps + if rm_deps: + update_opts["depends_on_remove"] = rm_deps + if set_deps is not None: + update_opts["depends_on_set"] = set_deps + if add_groups: + update_opts["groups_add"] = add_groups + if rm_groups: + update_opts["groups_remove"] = rm_groups + if add_key: + update_opts["keywords_add"] = add_key + if rm_key: + update_opts["keywords_remove"] = rm_key + if set_key is not None: + update_opts["keywords_set"] = set_key + if opt.os: + update_opts["op_sys"] = opt.os + if opt.arch: + update_opts["platform"] = opt.arch + if opt.priority: + update_opts["priority"] = opt.priority + if opt.qa_contact: + update_opts["qa_contact"] = opt.qa_contact + if opt.severity: + update_opts["severity"] = opt.severity + if status: + update_opts["status"] = status + if opt.summary: + update_opts["summary"] = opt.summary + if opt.version: + update_opts["version"] = opt.version + if opt.reset_assignee: + update_opts["reset_assigned_to"] = opt.reset_assignee + if opt.reset_qa_contact: + update_opts["reset_qa_contact"] = opt.reset_qa_contact + if opt.close: + update_opts["resolution"] = opt.close + if opt.target_release: + update_opts["target_release"] = opt.target_release + if opt.target_milestone: + update_opts["target_milestone"] = opt.target_milestone + if opt.dupeid: + update_opts["dupe_of"] = opt.dupeid + if opt.fixed_in: + update_opts["fixed_in"] = opt.fixed_in + if set_wb and set_wb[0]: + update_opts["whiteboard"] = set_wb and set_wb[0] + if set_devwb and set_devwb[0]: + update_opts["devel_whiteboard"] = set_devwb and set_devwb[0] + if set_intwb and set_intwb[0]: + update_opts["internal_whiteboard"] = set_intwb and set_intwb[0] + if set_qawb and set_qawb[0]: + update_opts["qa_whiteboard"] = set_qawb and set_qawb[0] + if opt.sub_component: + update_opts["sub_component"] = opt.sub_component + if opt.alias: + update_opts["alias"] = opt.alias + if flags: + update_opts["flags"] = flags + if opt.comment_tag: + update_opts["comment_tags"] = opt.comment_tag + if opt.minor_update: + update_opts["minor_update"] = opt.minor_update + + update = bz.build_update(**update_opts) # We make this a little convoluted to facilitate unit testing wbmap = { @@ -918,7 +1052,8 @@ def _do_modify(bz, parser, opt): if not v[0] and not v[1]: del(wbmap[k]) - _merge_field_opts(update, opt, parser) + if opt.fields: + _merge_field_opts(update, opt.fields, parser) log.debug("update bug dict=%s", update) log.debug("update whiteboard dict=%s", wbmap) @@ -926,9 +1061,6 @@ def _do_modify(bz, parser, opt): if not any([update, wbmap, add_tags, rm_tags]): parser.error("'modify' command requires additional arguments") - if opt.test_return_result: - return (update, wbmap, add_tags, rm_tags) - if add_tags or rm_tags: ret = bz.update_tags(bugid_list, tags_add=add_tags, tags_remove=rm_tags) @@ -943,33 +1075,49 @@ def _do_modify(bz, parser, opt): # Now for the things we can't blindly batch. # Being able to prepend/append to whiteboards, which are just # plain string values, is an old rhbz semantic that we try to maintain - # here. This is a bit weird for traditional bugzilla XMLRPC + # here. This is a bit weird for traditional bugzilla API log.debug("Adjusting whiteboard fields one by one") for bug in bz.getbugs(bugid_list): - for wb, (add_list, rm_list) in wbmap.items(): + update_kwargs = {} + for wbkey, (add_list, rm_list) in wbmap.items(): + bugval = getattr(bug, wbkey) or "" for tag in add_list: - newval = getattr(bug, wb) or "" - if newval: - newval += " " - newval += tag - bz.update_bugs([bug.id], - bz.build_update(**{wb: newval})) + if bugval: + bugval += " " + bugval += tag for tag in rm_list: - newval = (getattr(bug, wb) or "").split() - for t in newval[:]: + bugsplit = bugval.split() + for t in bugsplit[:]: if t == tag: - newval.remove(t) - bz.update_bugs([bug.id], - bz.build_update(**{wb: " ".join(newval)})) + bugsplit.remove(t) + bugval = " ".join(bugsplit) + + update_kwargs[wbkey] = bugval + + bz.update_bugs([bug.id], bz.build_update(**update_kwargs)) def _do_get_attach(bz, opt): - for bug in bz.getbugs(opt.getall): - opt.get += bug.get_attachment_ids() + data = {} + + def _process_attachment_data(_attlist): + for _att in _attlist: + data[_att["id"]] = _att + + if opt.getall: + for attlist in bz.get_attachments(opt.getall, None)["bugs"].values(): + _process_attachment_data(attlist) + if opt.get: + _process_attachment_data( + bz.get_attachments(None, opt.get)["attachments"].values()) + + for attdata in data.values(): + is_obsolete = attdata.get("is_obsolete", None) == 1 + if opt.ignore_obsolete and is_obsolete: + continue - for attid in set(opt.get): - att = bz.openattachment(attid) + att = bz.openattachment_data(attdata) outfile = open_without_clobber(att.name, "wb") data = att.read(4096) while data: @@ -977,8 +1125,6 @@ def _do_get_attach(bz, opt): data = att.read(4096) print("Wrote %s" % outfile.name) - return - def _do_set_attach(bz, opt, parser): if not opt.ids: @@ -1011,6 +1157,8 @@ def _do_set_attach(bz, opt, parser): kwargs["ispatch"] = True if opt.comment: kwargs["comment"] = opt.comment + if opt.private: + kwargs["is_private"] = True desc = opt.desc or os.path.basename(fileobj.name) # Upload attachments @@ -1032,17 +1180,19 @@ def _make_bz_instance(opt): cookiefile = None tokenfile = None + use_creds = False if opt.cache_credentials: cookiefile = opt.cookiefile or -1 tokenfile = opt.tokenfile or -1 + use_creds = True - bz = bugzilla.Bugzilla( + return bugzilla.Bugzilla( url=opt.bugzilla, cookiefile=cookiefile, tokenfile=tokenfile, sslverify=opt.sslverify, + use_creds=use_creds, cert=opt.cert) - return bz def _handle_login(opt, action, bz): @@ -1055,12 +1205,19 @@ def _handle_login(opt, action, bz): opt.login or opt.username or opt.password) username = getattr(opt, "pos_username", None) or opt.username password = getattr(opt, "pos_password", None) or opt.password + use_key = getattr(opt, "api_key", False) try: - if do_interactive_login: - if bz.url: - print("Logging into %s" % urlparse(bz.url)[1]) - bz.interactive_login(username, password) + if use_key: + bz.interactive_save_api_key() + elif do_interactive_login: + if bz.api_key: + print("You already have an API key configured for %s" % bz.url) + print("There is no need to cache a login token. Exiting.") + sys.exit(0) + print("Logging into %s" % urllib.parse.urlparse(bz.url)[1]) + bz.interactive_login(username, password, + restrict_login=opt.restrict_login) except bugzilla.BugzillaError as e: print(str(e)) sys.exit(1) @@ -1071,11 +1228,6 @@ def _handle_login(opt, action, bz): sys.exit(1) if is_login_command: - msg = "Login successful." - if bz.cookiefile or bz.tokenfile: - msg = "Login successful, token cache updated." - - print(msg) sys.exit(0) @@ -1086,9 +1238,7 @@ def _main(unittest_bz_instance): setup_logging(opt.debug, opt.verbose) log.debug("Launched with command line: %s", " ".join(sys.argv)) - - # Connect to bugzilla - log.info('Connecting to %s', opt.bugzilla) + log.debug("Bugzilla module: %s", bugzilla) if unittest_bz_instance: bz = unittest_bz_instance @@ -1104,28 +1254,18 @@ def _main(unittest_bz_instance): ########################### if hasattr(opt, "outputformat"): - if not opt.outputformat and opt.output not in ['raw', None]: + if not opt.outputformat and opt.output not in ['raw', 'json', None]: opt.outputformat = _convert_to_outputformat(opt.output) buglist = [] if action == 'info': - if not (opt.products or - opt.components or - opt.component_owners or - opt.versions): - parser.error("'info' command requires additional arguments") - _do_info(bz, opt) elif action == 'query': buglist = _do_query(bz, opt, parser) - if opt.test_return_result: - return buglist elif action == 'new': buglist = _do_new(bz, opt, parser) - if opt.test_return_result: - return buglist elif action == 'attach': if opt.get or opt.getall: @@ -1137,10 +1277,8 @@ def _main(unittest_bz_instance): _do_set_attach(bz, opt, parser) elif action == 'modify': - modout = _do_modify(bz, parser, opt) - if opt.test_return_result: - return modout - else: + _do_modify(bz, parser, opt) + else: # pragma: no cover raise RuntimeError("Unexpected action '%s'" % action) # If we're doing new/query/modify, output our results @@ -1155,7 +1293,10 @@ def main(unittest_bz_instance=None): except (Exception, KeyboardInterrupt): log.debug("", exc_info=True) raise - except (Fault, bugzilla.BugzillaError) as e: + except KeyboardInterrupt: + print("\nExited at user request.") + sys.exit(1) + except (xmlrpc.client.Fault, bugzilla.BugzillaError) as e: print("\nServer error: %s" % str(e)) sys.exit(3) except requests.exceptions.SSLError as e: @@ -1168,15 +1309,11 @@ def main(unittest_bz_instance=None): except (socket.error, requests.exceptions.HTTPError, requests.exceptions.ConnectionError, - ProtocolError) as e: + requests.exceptions.InvalidURL, + xmlrpc.client.ProtocolError) as e: print("\nConnection lost/failed: %s" % str(e)) sys.exit(2) def cli(): - try: - main() - except KeyboardInterrupt: - log.debug("", exc_info=True) - print("\nExited at user request.") - sys.exit(1) + main() diff --git a/scripts/bugzilla/_rhconverters.py b/scripts/bugzilla/_rhconverters.py new file mode 100644 index 0000000..fb371cd --- /dev/null +++ b/scripts/bugzilla/_rhconverters.py @@ -0,0 +1,128 @@ +# rhbugzilla.py - a Python interface to Red Hat Bugzilla using xmlrpclib. +# +# Copyright (C) 2008-2012 Red Hat Inc. +# Author: Will Woods +# +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +from logging import getLogger + +from ._util import listify + +log = getLogger(__name__) + + +class _RHBugzillaConverters(object): + """ + Static class that holds functional Red Hat back compat converters. + Called inline in Bugzilla + """ + @staticmethod + def convert_build_update( + component=None, + fixed_in=None, + qa_whiteboard=None, + devel_whiteboard=None, + internal_whiteboard=None, + sub_component=None): + adddict = {} + + def get_alias(): + # RHBZ has a custom extension to allow a bug to have multiple + # aliases, so the format of aliases is + # {"add": [...], "remove": [...]} + # But that means in order to approximate upstream, behavior + # which just overwrites the existing alias, we need to read + # the bug's state first to know what string to remove. Which + # we can't do, since we don't know the bug numbers at this point. + # So fail for now. + # + # The API should provide {"set": [...]} + # https://bugzilla.redhat.com/show_bug.cgi?id=1173114 + # + # Implementation will go here when it's available + pass + + if fixed_in is not None: + adddict["cf_fixed_in"] = fixed_in + if qa_whiteboard is not None: + adddict["cf_qa_whiteboard"] = qa_whiteboard + if devel_whiteboard is not None: + adddict["cf_devel_whiteboard"] = devel_whiteboard + if internal_whiteboard is not None: + adddict["cf_internal_whiteboard"] = internal_whiteboard + + if sub_component: + if not isinstance(sub_component, dict): + component = listify(component) + if not component: + raise ValueError("component must be specified if " + "specifying sub_component") + sub_component = {component[0]: sub_component} + adddict["sub_components"] = sub_component + + get_alias() + + return adddict + + + ################# + # Query methods # + ################# + + @staticmethod + def pre_translation(query): + """ + Translates the query for possible aliases + """ + old = query.copy() + + def split_comma(_v): + if isinstance(_v, list): + return _v + return _v.split(",") + + if 'bug_id' in query: + query['id'] = split_comma(query.pop('bug_id')) + + if 'component' in query: + query['component'] = split_comma(query['component']) + + if 'include_fields' not in query and 'column_list' in query: + query['include_fields'] = query.pop('column_list') + + if old != query: + log.debug("RHBugzilla pretranslated query to: %s", query) + + @staticmethod + def post_translation(query, bug): + """ + Convert the results of getbug back to the ancient RHBZ value + formats + """ + ignore = query + + # RHBZ _still_ returns component and version as lists, which + # deviates from upstream. Copy the list values to components + # and versions respectively. + if 'component' in bug and "components" not in bug: + val = bug['component'] + bug['components'] = isinstance(val, list) and val or [val] + bug['component'] = bug['components'][0] + + if 'version' in bug and "versions" not in bug: + val = bug['version'] + bug['versions'] = isinstance(val, list) and val or [val] + bug['version'] = bug['versions'][0] + + # sub_components isn't too friendly of a format, add a simpler + # sub_component value + if 'sub_components' in bug and 'sub_component' not in bug: + val = bug['sub_components'] + bug['sub_component'] = "" + if isinstance(val, dict): + values = [] + for vallist in val.values(): + values += vallist + bug['sub_component'] = " ".join(values) diff --git a/scripts/bugzilla/_session.py b/scripts/bugzilla/_session.py new file mode 100644 index 0000000..ce03051 --- /dev/null +++ b/scripts/bugzilla/_session.py @@ -0,0 +1,114 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +from logging import getLogger + +import os +import sys +import urllib.parse + +import requests + + +log = getLogger(__name__) + + +class _BugzillaSession(object): + """ + Class to handle the backend agnostic 'requests' setup + """ + def __init__(self, url, user_agent, + sslverify, cert, tokencache, api_key, + is_redhat_bugzilla, + requests_session=None): + self._url = url + self._user_agent = user_agent + self._scheme = urllib.parse.urlparse(url)[0] + self._tokencache = tokencache + self._api_key = api_key + self._is_xmlrpc = False + self._use_auth_bearer = False + + if self._scheme not in ["http", "https"]: + raise Exception("Invalid URL scheme: %s (%s)" % ( + self._scheme, url)) + + self._session = requests_session + if not self._session: + self._session = requests.Session() + + if cert: + self._session.cert = cert + if sslverify is False: + self._session.verify = False + self._session.headers["User-Agent"] = self._user_agent + + if is_redhat_bugzilla and self._api_key: + self._use_auth_bearer = True + self._session.headers["Authorization"] = ( + "Bearer %s" % self._api_key) + + def _get_timeout(self): + # Default to 5 minutes. This is longer than bugzilla.redhat.com's + # apparent 3 minute timeout so shouldn't affect legitimate usage, + # but saves us from indefinite hangs + DEFAULT_TIMEOUT = 300 + envtimeout = os.environ.get("PYTHONBUGZILLA_REQUESTS_TIMEOUT") + return float(envtimeout or DEFAULT_TIMEOUT) + + def set_rest_defaults(self): + self._session.headers["Content-Type"] = "application/json" + def set_xmlrpc_defaults(self): + self._is_xmlrpc = True + self._session.headers["Content-Type"] = "text/xml" + + def get_user_agent(self): + return self._user_agent + def get_scheme(self): + return self._scheme + + def get_auth_params(self): + # bugzilla.redhat.com will error if there's auth bits in params + # when Authorization header is used + if self._use_auth_bearer: + return {} + + # Don't add a token to the params list if an API key is set. + # Keeping API key solo means bugzilla will definitely fail + # if the key expires. Passing in a token could hide that + # fact, which could make it confusing to pinpoint the issue. + if self._api_key: + # Bugzilla 5.0 only supports api_key as a query parameter. + # Bugzilla 5.1+ takes it as a X-BUGZILLA-API-KEY header as well, + # with query param taking preference. + return {"Bugzilla_api_key": self._api_key} + + token = self._tokencache.get_value(self._url) + if token: + return {"Bugzilla_token": token} + + return {} + + def get_requests_session(self): + return self._session + + def request(self, *args, **kwargs): + timeout = self._get_timeout() + if "timeout" not in kwargs: + kwargs["timeout"] = timeout + + response = self._session.request(*args, **kwargs) + + if self._is_xmlrpc: + # Yes this still appears to matter for properly decoding unicode + # code points in bugzilla.redhat.com content + response.encoding = "UTF-8" + + try: + response.raise_for_status() + except Exception as e: + # Scrape the api key out of the returned exception string + message = str(e).replace(self._api_key or "", "") + raise type(e)(message).with_traceback(sys.exc_info()[2]) + + return response diff --git a/scripts/bugzilla/_util.py b/scripts/bugzilla/_util.py new file mode 100644 index 0000000..0455577 --- /dev/null +++ b/scripts/bugzilla/_util.py @@ -0,0 +1,12 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + + +def listify(val): + """Ensure that value is either None or a list, converting single values + into 1-element lists""" + if val is None: + return val + if isinstance(val, list): + return val + return [val] diff --git a/scripts/bugzilla/apiversion.py b/scripts/bugzilla/apiversion.py index 4e6e2c1..3a6d3e8 100644 --- a/scripts/bugzilla/apiversion.py +++ b/scripts/bugzilla/apiversion.py @@ -1,11 +1,8 @@ # # Copyright (C) 2014 Red Hat Inc. # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. -version = "2.2.0.dev0" +version = "3.2.0" __version__ = version diff --git a/scripts/bugzilla/base.py b/scripts/bugzilla/base.py index 483b0ee..68b3683 100644 --- a/scripts/bugzilla/base.py +++ b/scripts/bugzilla/base.py @@ -3,143 +3,44 @@ # Copyright (C) 2007, 2008, 2009, 2010 Red Hat Inc. # Author: Will Woods # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. import collections import getpass import locale from logging import getLogger +import mimetypes import os import sys +import urllib.parse from io import BytesIO -# pylint: disable=import-error -if sys.version_info[0] >= 3: - # pylint: disable=no-name-in-module - from configparser import SafeConfigParser - from http.cookiejar import LoadError, MozillaCookieJar - from urllib.parse import urlparse, parse_qsl - from xmlrpc.client import Binary, Fault -else: - from ConfigParser import SafeConfigParser - from cookielib import LoadError, MozillaCookieJar - from urlparse import urlparse, parse_qsl - from xmlrpclib import Binary, Fault -# pylint: enable=import-error - - +from ._authfiles import _BugzillaRCFile, _BugzillaTokenCache from .apiversion import __version__ -from .bug import Bug, User -from .transport import BugzillaError, _BugzillaServerProxy, _RequestsTransport +from ._backendrest import _BackendREST +from ._backendxmlrpc import _BackendXMLRPC +from .bug import Bug, Group, User +from .exceptions import BugzillaError +from ._rhconverters import _RHBugzillaConverters +from ._session import _BugzillaSession +from ._util import listify log = getLogger(__name__) -mimemagic = None - - -def _detect_filetype(fname): - global mimemagic - - if mimemagic is None: - try: - # pylint: disable=import-error - import magic - mimemagic = magic.open(getattr(magic, "MAGIC_MIME_TYPE", 16)) - mimemagic.load() - except ImportError as e: - log.debug("Could not load python-magic: %s", e) - mimemagic = None - if not mimemagic: - return None - - if not os.path.isabs(fname): - return None - - try: - return mimemagic.file(fname) - except Exception as e: - log.debug("Could not detect content_type: %s", e) - return None - def _nested_update(d, u): # Helper for nested dict update() - # https://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth for k, v in list(u.items()): - if isinstance(v, collections.Mapping): + if isinstance(v, collections.abc.Mapping): d[k] = _nested_update(d.get(k, {}), v) else: d[k] = v return d -def _default_auth_location(filename): - """ - Determine auth location for filename, like 'bugzillacookies'. If - old style ~/.bugzillacookies exists, we use that, otherwise we - use ~/.cache/python-bugzilla/bugzillacookies. Same for bugzillatoken - """ - homepath = os.path.expanduser("~/.%s" % filename) - xdgpath = os.path.expanduser("~/.cache/python-bugzilla/%s" % filename) - if os.path.exists(xdgpath): - return xdgpath - if os.path.exists(homepath): - return homepath - - if not os.path.exists(os.path.dirname(xdgpath)): - os.makedirs(os.path.dirname(xdgpath), 0o700) - return xdgpath - - -def _build_cookiejar(cookiefile): - cj = MozillaCookieJar(cookiefile) - if cookiefile is None: - return cj - if not os.path.exists(cookiefile): - # Make sure a new file has correct permissions - open(cookiefile, 'a').close() - os.chmod(cookiefile, 0o600) - cj.save() - return cj - - try: - cj.load() - return cj - except LoadError: - raise BugzillaError("cookiefile=%s not in Mozilla format" % - cookiefile) - - -_default_configpaths = [ - '/etc/bugzillarc', - '~/.bugzillarc', - '~/.config/python-bugzilla/bugzillarc', -] - - -def _open_bugzillarc(configpaths=-1): - if configpaths == -1: - configpaths = _default_configpaths[:] - - # pylint: disable=protected-access - configpaths = [os.path.expanduser(p) for p in - Bugzilla._listify(configpaths)] - # pylint: enable=protected-access - cfg = SafeConfigParser() - read_files = cfg.read(configpaths) - if not read_files: - return - - log.info("Found bugzillarc files: %s", read_files) - return cfg - - class _FieldAlias(object): """ Track API attribute names that differ from what we expose in users. @@ -171,6 +72,8 @@ class _BugzillaAPICache(object): self.products = [] self.component_names = {} self.bugfields = [] + self.version_raw = None + self.version_parsed = (0, 0) class Bugzilla(object): @@ -183,11 +86,11 @@ class Bugzilla(object): bzapi = Bugzilla("http://bugzilla.example.com") If you have previously logged into that URL, and have cached login - cookies/tokens, you will automatically be logged in. Otherwise to + tokens, you will automatically be logged in. Otherwise to log in, you can either pass auth options to __init__, or call a login helper like interactive_login(). - If you are not logged in, you won be able to access restricted data like + If you are not logged in, you won't be able to access restricted data like user email, or perform write actions like bug create/update. But simple querys will work correctly. @@ -197,29 +100,23 @@ class Bugzilla(object): Another way to specify auth credentials is via a 'bugzillarc' file. See readconfig() documentation for details. """ - - # bugzilla version that the class is targeting. filled in by - # subclasses - bz_ver_major = 0 - bz_ver_minor = 0 - @staticmethod def url_to_query(url): - ''' + """ Given a big huge bugzilla query URL, returns a query dict that can be passed along to the Bugzilla.query() method. - ''' + """ q = {} # pylint: disable=unpacking-non-sequence - (ignore, ignore, path, - ignore, query, ignore) = urlparse(url) + (ignore1, ignore2, path, + ignore, query, ignore3) = urllib.parse.urlparse(url) base = os.path.basename(path) if base not in ('buglist.cgi', 'query.cgi'): return {} - for (k, v) in parse_qsl(query): + for (k, v) in urllib.parse.parse_qsl(query): if k not in q: q[k] = v elif isinstance(q[k], list): @@ -238,30 +135,46 @@ class Bugzilla(object): return q @staticmethod - def fix_url(url): + def fix_url(url, force_rest=False): """ Turn passed url into a bugzilla XMLRPC web url + + :param force_rest: If True, generate a REST API url """ - if '://' not in url: - log.debug('No scheme given for url, assuming https') - url = 'https://' + url - if url.count('/') < 3: - log.debug('No path given for url, assuming /xmlrpc.cgi') - url = url + '/xmlrpc.cgi' - return url + (scheme, netloc, path, + params, query, fragment) = urllib.parse.urlparse(url) + if not scheme: + scheme = 'https' + + if path and not netloc: + netloc = path.split("/", 1)[0] + path = "/".join(path.split("/")[1:]) or None + + if not path: + path = 'xmlrpc.cgi' + if force_rest: + path = "rest/" + + newurl = urllib.parse.urlunparse( + (scheme, netloc, path, params, query, fragment)) + return newurl @staticmethod - def _listify(val): - if val is None: - return val - if isinstance(val, list): - return val - return [val] + def get_rcfile_default_url(): + """ + Helper to check all the default bugzillarc file paths for + a [DEFAULT] url=X section, and if found, return it. + """ + configpaths = _BugzillaRCFile.get_default_configpaths() + rcfile = _BugzillaRCFile() + rcfile.set_configpaths(configpaths) + return rcfile.get_default_url() def __init__(self, url=-1, user=None, password=None, cookiefile=-1, sslverify=True, tokenfile=-1, use_creds=True, api_key=None, - cert=None, authtype=None): + cert=None, configpaths=-1, + force_rest=False, force_xmlrpc=False, requests_session=None): """ :param url: The bugzilla instance URL, which we will connect to immediately. Most users will want to specify this at @@ -271,25 +184,29 @@ class Bugzilla(object): :param password: optional password for the connecting user :param cert: optional certificate file for client side certificate authentication - :param cookiefile: Location to cache the login session cookies so you - don't have to keep specifying username/password. Bugzilla 5+ will - use tokens instead of cookies. - If -1, use the default path. If None, don't use or save - any cookiefile. + :param cookiefile: Deprecated, raises an error if not -1 or None :param sslverify: Set this to False to skip SSL hostname and CA validation checks, like out of date certificate :param tokenfile: Location to cache the API login token so youi don't have to keep specifying username/password. If -1, use the default path. If None, don't use or save any tokenfile. - :param use_creds: If False, this disables cookiefile, tokenfile, - and any bugzillarc reading. This overwrites any tokenfile - or cookiefile settings + :param use_creds: If False, this disables tokenfile + and configpaths by default. This is a convenience option to + unset those values at init time. If those values are later + changed, they may be used for future operations. :param sslverify: Maps to 'requests' sslverify parameter. Set to False to disable SSL verification, but it can also be a path to file or directory for custom certs. - :param api_key: A bugzilla - :param authtype: Authentication type: empty or 'basic' + :param api_key: A bugzilla5+ API key + :param configpaths: A list of possible bugzillarc locations. + :param force_rest: Force use of the REST API + :param force_xmlrpc: Force use of the XMLRPC API. If neither force_X + parameter are specified, heuristics will be used to determine + which API to use, with XMLRPC preferred for back compatability. + :param requests_session: An optional requests.Session object the + API will use to contact the remote bugzilla instance. This + way the API user can set up whatever auth bits they may need. """ if url == -1: raise TypeError("Specify a valid bugzilla url, or pass url=None") @@ -298,158 +215,177 @@ class Bugzilla(object): self.user = user or '' self.password = password or '' self.api_key = api_key - self.cert = cert or '' + self.cert = cert or None self.url = '' - self.authtype = authtype or '' - self._proxy = None - self._transport = None - self._cookiejar = None + self._backend = None + self._session = None + self._user_requests_session = requests_session self._sslverify = sslverify self._cache = _BugzillaAPICache() self._bug_autorefresh = False + self._is_redhat_bugzilla = False + + self._rcfile = _BugzillaRCFile() + self._tokencache = _BugzillaTokenCache() - self._field_aliases = [] - self._init_field_aliases() + self._force_rest = force_rest + self._force_xmlrpc = force_xmlrpc + + if cookiefile not in [None, -1]: + raise TypeError("cookiefile is deprecated, don't pass any value.") - self.configpath = _default_configpaths[:] if not use_creds: - cookiefile = None tokenfile = None - self.configpath = [] + configpaths = [] - if cookiefile == -1: - cookiefile = _default_auth_location("bugzillacookies") if tokenfile == -1: - tokenfile = _default_auth_location("bugzillatoken") - log.debug("Using tokenfile=%s", tokenfile) - self.cookiefile = cookiefile - self.tokenfile = tokenfile + tokenfile = self._tokencache.get_default_path() + if configpaths == -1: + configpaths = _BugzillaRCFile.get_default_configpaths() + + self._settokenfile(tokenfile) + self._setconfigpath(configpaths) if url: self.connect(url) - self._init_class_from_url() - self._init_class_state() + + def _detect_is_redhat_bugzilla(self): + if self._is_redhat_bugzilla: + return True + + match = ".redhat.com" + if match in self.url: + log.info("Using RHBugzilla for URL containing %s", match) + return True + + return False def _init_class_from_url(self): """ Detect if we should use RHBugzilla class, and if so, set it """ - from bugzilla import RHBugzilla - if isinstance(self, RHBugzilla): - return + from .oldclasses import RHBugzilla # pylint: disable=cyclic-import - c = None - if "bugzilla.redhat.com" in self.url: - log.info("Using RHBugzilla for URL containing bugzilla.redhat.com") - c = RHBugzilla - else: - try: - extensions = self._proxy.Bugzilla.extensions() - if "RedHat" in extensions.get('extensions', {}): - log.info("Found RedHat bugzilla extension, " - "using RHBugzilla") - c = RHBugzilla - except Fault: - log.debug("Failed to fetch bugzilla extensions", exc_info=True) - - if not c: + if not self._detect_is_redhat_bugzilla(): return - self.__class__ = c + self._is_redhat_bugzilla = True + if self.__class__ == Bugzilla: + # Overriding the class doesn't have any functional effect, + # but we continue to do it for API back compat incase anyone + # is doing any class comparison. We should drop this in the future + self.__class__ = RHBugzilla - def _init_class_state(self): - """ - Hook for subclasses to do any __init__ time setup - """ - pass - - def _init_field_aliases(self): + def _get_field_aliases(self): # List of field aliases. Maps old style RHBZ parameter # names to actual upstream values. Used for createbug() and # query include_fields at least. - self._add_field_alias('summary', 'short_desc') - self._add_field_alias('description', 'comment') - self._add_field_alias('platform', 'rep_platform') - self._add_field_alias('severity', 'bug_severity') - self._add_field_alias('status', 'bug_status') - self._add_field_alias('id', 'bug_id') - self._add_field_alias('blocks', 'blockedby') - self._add_field_alias('blocks', 'blocked') - self._add_field_alias('depends_on', 'dependson') - self._add_field_alias('creator', 'reporter') - self._add_field_alias('url', 'bug_file_loc') - self._add_field_alias('dupe_of', 'dupe_id') - self._add_field_alias('dupe_of', 'dup_id') - self._add_field_alias('comments', 'longdescs') - self._add_field_alias('creation_time', 'opendate') - self._add_field_alias('creation_time', 'creation_ts') - self._add_field_alias('whiteboard', 'status_whiteboard') - self._add_field_alias('last_change_time', 'delta_ts') + ret = [] + + def _add(*args, **kwargs): + ret.append(_FieldAlias(*args, **kwargs)) + + def _add_both(newname, origname): + _add(newname, origname, is_api=False) + _add(origname, newname, is_bug=False) + + _add('summary', 'short_desc') + _add('description', 'comment') + _add('platform', 'rep_platform') + _add('severity', 'bug_severity') + _add('status', 'bug_status') + _add('id', 'bug_id') + _add('blocks', 'blockedby') + _add('blocks', 'blocked') + _add('depends_on', 'dependson') + _add('creator', 'reporter') + _add('url', 'bug_file_loc') + _add('dupe_of', 'dupe_id') + _add('dupe_of', 'dup_id') + _add('comments', 'longdescs') + _add('creation_time', 'opendate') + _add('creation_time', 'creation_ts') + _add('whiteboard', 'status_whiteboard') + _add('last_change_time', 'delta_ts') + + if self._is_redhat_bugzilla: + _add_both('fixed_in', 'cf_fixed_in') + _add_both('qa_whiteboard', 'cf_qa_whiteboard') + _add_both('devel_whiteboard', 'cf_devel_whiteboard') + _add_both('internal_whiteboard', 'cf_internal_whiteboard') + + _add('component', 'components', is_bug=False) + _add('version', 'versions', is_bug=False) + # Yes, sub_components is the field name the API expects + _add('sub_components', 'sub_component', is_bug=False) + # flags format isn't exactly the same but it's the closest approx + _add('flags', 'flag_types') + + return ret def _get_user_agent(self): return 'python-bugzilla/%s' % __version__ user_agent = property(_get_user_agent) + @property + def bz_ver_major(self): + return self._cache.version_parsed[0] + + @property + def bz_ver_minor(self): + return self._cache.version_parsed[1] + ################### # Private helpers # ################### - def _check_version(self, major, minor): + def _get_version(self): """ - Check if the detected bugzilla version is >= passed major/minor pair. + Return version number as a float """ - if major < self.bz_ver_major: - return True - if (major == self.bz_ver_major and minor <= self.bz_ver_minor): - return True - return False - - def _add_field_alias(self, *args, **kwargs): - self._field_aliases.append(_FieldAlias(*args, **kwargs)) + return float("%d.%d" % (self.bz_ver_major, self.bz_ver_minor)) def _get_bug_aliases(self): return [(f.newname, f.oldname) - for f in self._field_aliases if f.is_bug] + for f in self._get_field_aliases() if f.is_bug] def _get_api_aliases(self): return [(f.newname, f.oldname) - for f in self._field_aliases if f.is_api] + for f in self._get_field_aliases() if f.is_api] - ################### - # Cookie handling # - ################### + ################# + # Auth handling # + ################# def _getcookiefile(self): - '''cookiefile is the file that bugzilla session cookies are loaded - and saved from. - ''' - return self._cookiejar.filename - - def _delcookiefile(self): - self._cookiejar = None - - def _setcookiefile(self, cookiefile): - if (self._cookiejar and cookiefile == self._cookiejar.filename): - return - - if self._proxy is not None: - raise RuntimeError("Can't set cookies with an open connection, " - "disconnect() first.") + return None + cookiefile = property(_getcookiefile) - log.debug("Using cookiefile=%s", cookiefile) - self._cookiejar = _build_cookiejar(cookiefile) + def _gettokenfile(self): + return self._tokencache.get_filename() + def _settokenfile(self, filename): + self._tokencache.set_filename(filename) + def _deltokenfile(self): + self._settokenfile(None) + tokenfile = property(_gettokenfile, _settokenfile, _deltokenfile) - cookiefile = property(_getcookiefile, _setcookiefile, _delcookiefile) + def _getconfigpath(self): + return self._rcfile.get_configpaths() + def _setconfigpath(self, configpaths): + return self._rcfile.set_configpaths(configpaths) + def _delconfigpath(self): + return self._rcfile.set_configpaths(None) + configpath = property(_getconfigpath, _setconfigpath, _delconfigpath) ############################# # Login/connection handling # ############################# - def readconfig(self, configpath=None): + def readconfig(self, configpath=None, overwrite=True): """ :param configpath: Optional bugzillarc path to read, instead of the default list. @@ -478,55 +414,73 @@ class Bugzilla(object): Be sure to set appropriate permissions on bugzillarc if you choose to store your password in it! - """ - cfg = _open_bugzillarc(configpath or self.configpath) - if not cfg: - return - section = "" - log.debug("bugzillarc: Searching for config section matching %s", - self.url) - for s in sorted(cfg.sections()): - # Substring match - prefer the longest match found - if s in self.url: - log.debug("bugzillarc: Found matching section: %s", s) - section = s - - if not section: - log.debug("bugzillarc: No section found") - return + :param overwrite: If True, bugzillarc will clobber any already + set self.user/password/api_key/cert value. + """ + if configpath: + self._setconfigpath(configpath) + data = self._rcfile.parse(self.url) - for key, val in cfg.items(section): - if key == "api_key": + for key, val in data.items(): + if key == "api_key" and (overwrite or not self.api_key): log.debug("bugzillarc: setting api_key") self.api_key = val - elif key == "user": + elif key == "user" and (overwrite or not self.user): log.debug("bugzillarc: setting user=%s", val) self.user = val - elif key == "password": + elif key == "password" and (overwrite or not self.password): log.debug("bugzillarc: setting password") self.password = val - elif key == "cert": + elif key == "cert" and (overwrite or not self.cert): log.debug("bugzillarc: setting cert") self.cert = val - elif key == "authtype": - log.debug("bugzillarc: setting authtype=%s", val) - self.authtype = val else: log.debug("bugzillarc: unknown key=%s", key) def _set_bz_version(self, version): + self._cache.version_raw = version try: - self.bz_ver_major, self.bz_ver_minor = [ - int(i) for i in version.split(".")[0:2]] + major, minor = [int(i) for i in version.split(".")[0:2]] except Exception: log.debug("version doesn't match expected format X.Y.Z, " "assuming 5.0", exc_info=True) - self.bz_ver_major = 5 - self.bz_ver_minor = 0 + major = 5 + minor = 0 + self._cache.version_parsed = (major, minor) + + def _get_backend_class(self, url): # pragma: no cover + # This is a hook for the test suite to do some mock hackery + if self._force_rest and self._force_xmlrpc: + raise BugzillaError( + "Cannot specify both force_rest and force_xmlrpc") + + xmlurl = self.fix_url(url) + if self._force_xmlrpc: + return _BackendXMLRPC, xmlurl + + resturl = self.fix_url(url, force_rest=self._force_rest) + if self._force_rest: + return _BackendREST, resturl + + # Simple heuristic if the original url has a path in it + if "/xmlrpc" in url: + return _BackendXMLRPC, xmlurl + if "/rest" in url: + return _BackendREST, resturl + + # We were passed something like bugzilla.example.com but we + # aren't sure which method to use, try probing + if _BackendXMLRPC.probe(xmlurl): + return _BackendXMLRPC, xmlurl + if _BackendREST.probe(resturl): + return _BackendREST, resturl + + # Otherwise fallback to XMLRPC default and let it fail + return _BackendXMLRPC, xmlurl def connect(self, url=None): - ''' + """ Connect to the bugzilla instance with the given url. This is called by __init__ if a URL is passed. Or it can be called manually at any time with a passed URL. @@ -536,57 +490,87 @@ class Bugzilla(object): If 'user' and 'password' are both set, we'll run login(). Otherwise you'll have to login() yourself before some methods will work. - ''' - if self._transport: + """ + if self._session: self.disconnect() - if url is None and self.url: - url = self.url - url = self.fix_url(url) + url = url or self.url + backendclass, newurl = self._get_backend_class(url) + if url != newurl: + log.debug("Converted url=%s to fixed url=%s", url, newurl) + self.url = newurl + log.debug("Connecting with URL %s", self.url) - self.url = url # we've changed URLs - reload config - self.readconfig() + self.readconfig(overwrite=False) - self._transport = _RequestsTransport( - url, self._cookiejar, sslverify=self._sslverify, cert=self.cert) - if self.authtype == 'basic' and self.user and self.password: - self._transport.session.auth = (self.user, self.password) - self._transport.user_agent = self.user_agent - self._proxy = _BugzillaServerProxy(url, self.tokenfile, - self._transport) + # Detect if connecting to redhat bugzilla + self._init_class_from_url() - if (self.authtype == '' and self.user and self.password): + self._session = _BugzillaSession(self.url, self.user_agent, + sslverify=self._sslverify, + cert=self.cert, + tokencache=self._tokencache, + api_key=self.api_key, + is_redhat_bugzilla=self._is_redhat_bugzilla, + requests_session=self._user_requests_session) + self._backend = backendclass(self.url, self._session) + + if (self.user and self.password): log.info("user and password present - doing login()") self.login() if self.api_key: log.debug("using API key") - self._proxy.use_api_key(self.api_key) - version = self._proxy.Bugzilla.version()["version"] + version = self._backend.bugzilla_version()["version"] log.debug("Bugzilla version string: %s", version) self._set_bz_version(version) - def disconnect(self): - ''' - Disconnect from the given bugzilla instance. - ''' - self._proxy = None - self._transport = None - self._cache = _BugzillaAPICache() + @property + def _proxy(self): + """ + Return an xmlrpc ServerProxy instance that will work seamlessly + with bugzilla + + Some apps have historically accessed _proxy directly, like + fedora infrastrucutre pieces. So we consider it part of the API + """ + return self._backend.get_xmlrpc_proxy() + + def is_xmlrpc(self): + """ + :returns: True if using the XMLRPC API + """ + return self._backend.is_xmlrpc() + + def is_rest(self): + """ + :returns: True if using the REST API + """ + return self._backend.is_rest() + + def get_requests_session(self): + """ + Give API users access to the Requests.session object we use for + talking to the remote bugzilla instance. - def _login(self, user, password): - '''Backend login method for Bugzilla3''' - return self._proxy.User.login({'login': user, 'password': password}) + :returns: The Requests.session object backing the open connection. + """ + return self._session.get_requests_session() - def _logout(self): - '''Backend login method for Bugzilla3''' - return self._proxy.User.logout() + def disconnect(self): + """ + Disconnect from the given bugzilla instance. + """ + self._backend = None + self._session = None + self._cache = _BugzillaAPICache() - def login(self, user=None, password=None): - '''Attempt to log in using the given username and password. Subsequent + def login(self, user=None, password=None, restrict_login=None): + """ + Attempt to log in using the given username and password. Subsequent method calls will use this username and password. Returns False if login fails, otherwise returns some kind of login info - typically either a numeric userid, or a dict of user info. @@ -595,10 +579,13 @@ class Bugzilla(object): is not set, ValueError will be raised. If login fails, BugzillaError will be raised. + The login session can be restricted to current user IP address + with restrict_login argument. (Bugzilla 4.4+) + This method will be called implicitly at the end of connect() if user and password are both set. So under most circumstances you won't need to call this yourself. - ''' + """ if self.api_key: raise ValueError("cannot login when using an API key") @@ -612,21 +599,61 @@ class Bugzilla(object): if not self.password: raise ValueError("missing password") + payload = {"login": self.user} + if restrict_login: + payload['restrict_login'] = True + log.debug("logging in with options %s", str(payload)) + payload['password'] = self.password + try: - ret = self._login(self.user, self.password) + ret = self._backend.user_login(payload) self.password = '' - log.info("login successful for user=%s", self.user) + log.info("login succeeded for user=%s", self.user) + if "token" in ret: + self._tokencache.set_value(self.url, ret["token"]) return ret - except Fault as e: - raise BugzillaError("Login failed: %s" % str(e.faultString)) + except Exception as e: + log.debug("Login exception: %s", str(e), exc_info=True) + raise BugzillaError("Login failed: %s" % + BugzillaError.get_bugzilla_error_string(e)) from None + + def interactive_save_api_key(self): + """ + Helper method to interactively ask for an API key, verify it + is valid, and save it to a bugzillarc file referenced via + self.configpaths + """ + sys.stdout.write('API Key: ') + sys.stdout.flush() + api_key = sys.stdin.readline().strip() + + self.disconnect() + self.api_key = api_key + + log.info('Checking API key... ') + self.connect() + + if not self.logged_in: # pragma: no cover + raise BugzillaError("Login with API_KEY failed") + log.info('API Key accepted') - def interactive_login(self, user=None, password=None, force=False): + wrote_filename = self._rcfile.save_api_key(self.url, self.api_key) + log.info("API key written to filename=%s", wrote_filename) + + msg = "Login successful." + if wrote_filename: + msg += " API key written to %s" % wrote_filename + print(msg) + + def interactive_login(self, user=None, password=None, force=False, + restrict_login=None): """ Helper method to handle login for this bugzilla instance. :param user: bugzilla username. If not specified, prompt for it. :param password: bugzilla password. If not specified, prompt for it. :param force: Unused + :param restrict_login: restricts session to IP address """ ignore = force log.debug('Calling interactive_login') @@ -639,13 +666,27 @@ class Bugzilla(object): password = getpass.getpass('Bugzilla Password: ') log.info('Logging in... ') - self.login(user, password) - log.info('Authorization cookie received.') + out = self.login(user, password, restrict_login) + msg = "Login successful." + if "token" not in out: + msg += " However no token was returned." + else: + if not self.tokenfile: + msg += " Token not saved to disk." + else: + msg += " Token cache saved to %s" % self.tokenfile + if self._get_version() >= 5.0: + msg += "\nToken usage is deprecated. " + msg += "Consider using bugzilla API keys instead. " + msg += "See `man bugzilla` for more details." + print(msg) def logout(self): - '''Log out of bugzilla. Drops server connection and user info, and - destroys authentication cookies.''' - self._logout() + """ + Log out of bugzilla. Drops server connection and user info, and + destroys authentication cache + """ + self._backend.user_logout() self.disconnect() self.user = '' self.password = '' @@ -670,10 +711,11 @@ class Bugzilla(object): http://bugzilla.readthedocs.org/en/latest/api/core/v1/user.html#valid-login """ try: - self._proxy.User.get({'ids': []}) + self._backend.user_get({"ids": [1]}) return True - except Fault as e: - if e.faultCode == 505 or e.faultCode == 32000: + except Exception as e: + code = BugzillaError.get_bugzilla_error_code(e) + if code in [505, 32000]: return False raise e @@ -682,22 +724,26 @@ class Bugzilla(object): # Bugfields querying # ###################### - def _getbugfields(self): - ''' - Get the list of valid fields for Bug objects - ''' - r = self._proxy.Bug.fields({'include_fields': ['name']}) - return [f['name'] for f in r['fields']] - - def getbugfields(self, force_refresh=False): - ''' + def getbugfields(self, force_refresh=False, names=None): + """ Calls getBugFields, which returns a list of fields in each bug for this bugzilla instance. This can be used to set the list of attrs on the Bug object. - ''' + + :param force_refresh: If True, overwrite the bugfield cache + with these newly checked values. + :param names: Only check for the passed bug field names + """ + def _fieldnames(): + data = {"include_fields": ["name"]} + if names: + data["names"] = names + r = self._backend.bug_fields(data) + return [f['name'] for f in r['fields']] + if force_refresh or not self._cache.bugfields: log.debug("Refreshing bugfields") - self._cache.bugfields = self._getbugfields() + self._cache.bugfields = _fieldnames() self._cache.bugfields.sort() log.debug("bugfields = %s", self._cache.bugfields) @@ -734,11 +780,11 @@ class Bugzilla(object): if ptype: raw = None if ptype == "accessible": - raw = self._proxy.Product.get_accessible_products() - elif ptype == "selectable": - raw = self._proxy.Product.get_selectable_products() + raw = self._backend.product_get_accessible() elif ptype == "enterable": - raw = self._proxy.Product.get_enterable_products() + raw = self._backend.product_get_enterable() + elif ptype == "selectable": + raw = self._backend.product_get_selectable() if raw is None: raise RuntimeError("Unknown ptype=%s" % ptype) @@ -747,16 +793,15 @@ class Bugzilla(object): kwargs = {} if ids: - kwargs["ids"] = self._listify(ids) + kwargs["ids"] = listify(ids) if names: - kwargs["names"] = self._listify(names) + kwargs["names"] = listify(names) if include_fields: kwargs["include_fields"] = include_fields if exclude_fields: kwargs["exclude_fields"] = exclude_fields - log.debug("Calling Product.get with: %s", kwargs) - ret = self._proxy.Product.get(kwargs) + ret = self._backend.product_get(kwargs) return ret['products'] def refresh_products(self, **kwargs): @@ -857,12 +902,6 @@ class Bugzilla(object): """ Return a list of component names for the passed product. - This can be implemented with Product.get, but behind the - scenes it uses Bug.legal_values. Reason being that on bugzilla - instances with tons of components, like bugzilla.redhat.com - Product=Fedora for example, there's a 10x speed difference - even with properly limited Product.get calls. - On first invocation the value is cached, and subsequent calls will return the cached data. @@ -872,17 +911,22 @@ class Bugzilla(object): proddict = self._lookup_product_in_cache(product) product_id = proddict.get("id", None) - if (force_refresh or - product_id is None or - product_id not in self._cache.component_names): - self.refresh_products(names=[product], - include_fields=["names", "id"]) + if (force_refresh or product_id is None or + "components" not in proddict): + self.refresh_products( + names=[product], + include_fields=["name", "id", "components.name"]) proddict = self._lookup_product_in_cache(product) + if "id" not in proddict: + raise BugzillaError("Product '%s' not found" % product) product_id = proddict["id"] - opts = {'product_id': product_id, 'field': 'component'} - log.debug("Calling Bug.legal_values with: %s", opts) - names = self._proxy.Bug.legal_values(opts)["values"] + if product_id not in self._cache.component_names: + names = [] + for comp in proddict.get("components", []): + name = comp.get("name") + if name: + names.append(name) self._cache.component_names[product_id] = names return self._cache.component_names[product_id] @@ -915,13 +959,13 @@ class Bugzilla(object): def addcomponent(self, data): - ''' + """ A method to create a component in Bugzilla. Takes a dict, with the following elements: product: The product to create the component in component: The name of the component to create - desription: A one sentence summary of the component + description: A one sentence summary of the component default_assignee: The bugzilla login (email address) of the initial owner of the component default_qa_contact (optional): The bugzilla login of the @@ -930,23 +974,21 @@ class Bugzilla(object): new bugs for the component. is_active: (optional) If False, the component is hidden from the component list when filing new bugs. - ''' + """ data = data.copy() self._component_data_convert(data) - log.debug("Calling Component.create with: %s", data) - return self._proxy.Component.create(data) + return self._backend.component_create(data) def editcomponent(self, data): - ''' + """ A method to edit a component in Bugzilla. Takes a dict, with mandatory elements of product. component, and initialowner. All other elements are optional and use the same names as the addcomponent() method. - ''' + """ data = data.copy() self._component_data_convert(data, update=True) - log.debug("Calling Component.update with: %s", data) - return self._proxy.Component.update(data) + return self._backend.component_update(data) ################### @@ -959,9 +1001,6 @@ class Bugzilla(object): Internal helper to process include_fields lists """ def _convert_fields(_in): - if not _in: - return _in - for newname, oldname in self._get_api_aliases(): if oldname in _in: _in.remove(oldname) @@ -970,16 +1009,15 @@ class Bugzilla(object): return _in ret = {} - if self._check_version(4, 0): - if include_fields: - include_fields = _convert_fields(include_fields) - if "id" not in include_fields: - include_fields.append("id") - ret["include_fields"] = include_fields - if exclude_fields: - exclude_fields = _convert_fields(exclude_fields) - ret["exclude_fields"] = exclude_fields - if self._supports_getbug_extra_fields: + if include_fields: + include_fields = _convert_fields(include_fields) + if "id" not in include_fields: + include_fields.append("id") + ret["include_fields"] = include_fields + if exclude_fields: + exclude_fields = _convert_fields(exclude_fields) + ret["exclude_fields"] = exclude_fields + if self._supports_getbug_extra_fields(): if extra_fields: ret["extra_fields"] = _convert_fields(extra_fields) return ret @@ -997,61 +1035,78 @@ class Bugzilla(object): bug_autorefresh = property(_get_bug_autorefresh, _set_bug_autorefresh) - # getbug_extra_fields: Extra fields that need to be explicitly - # requested from Bug.get in order for the data to be returned. - # - # As of Dec 2012 it seems like only RH bugzilla actually has behavior - # like this, for upstream bz it returns all info for every Bug.get() - _getbug_extra_fields = [] - _supports_getbug_extra_fields = False + def _getbug_extra_fields(self): + """ + Extra fields that need to be explicitly + requested from Bug.get in order for the data to be returned. + """ + rhbz_extra_fields = [ + "comments", "description", + "external_bugs", "flags", "sub_components", + "tags", + ] + if self._is_redhat_bugzilla: + return rhbz_extra_fields + return [] + + def _supports_getbug_extra_fields(self): + """ + Return True if the bugzilla instance supports passing + extra_fields to getbug + + As of Dec 2012 it seems like only RH bugzilla actually has behavior + like this, for upstream bz it returns all info for every Bug.get() + """ + return self._is_redhat_bugzilla + def _getbugs(self, idlist, permissive, include_fields=None, exclude_fields=None, extra_fields=None): - ''' + """ Return a list of dicts of full bug info for each given bug id. bug ids that couldn't be found will return None instead of a dict. - ''' - oldidlist = idlist - idlist = [] - for i in oldidlist: - try: - idlist.append(int(i)) - except ValueError: - # String aliases can be passed as well - idlist.append(i) - - extra_fields = self._listify(extra_fields or []) - extra_fields += self._getbug_extra_fields - - getbugdata = {"ids": idlist} + """ + ids = [] + aliases = [] + + def _alias_or_int(_v): + if str(_v).isdigit(): + return int(_v), None + return None, str(_v) + + for idstr in idlist: + idint, alias = _alias_or_int(idstr) + if alias: + aliases.append(alias) + else: + ids.append(idstr) + + extra_fields = listify(extra_fields or []) + extra_fields += self._getbug_extra_fields() + + getbugdata = {} if permissive: getbugdata["permissive"] = 1 getbugdata.update(self._process_include_fields( include_fields, exclude_fields, extra_fields)) - log.debug("Calling Bug.get with: %s", getbugdata) - r = self._proxy.Bug.get(getbugdata) - - if self._check_version(4, 0): - bugdict = dict([(b['id'], b) for b in r['bugs']]) - else: - bugdict = dict([(b['id'], b['internals']) for b in r['bugs']]) + r = self._backend.bug_get(ids, aliases, getbugdata) + # Do some wrangling to ensure we return bugs in the same order + # the were passed in, for historical reasons ret = [] - for i in idlist: - found = None - if i in bugdict: - found = bugdict[i] - else: - # Need to map an alias - for valdict in bugdict.values(): - if i in self._listify(valdict.get("alias", None)): - found = valdict - break - - ret.append(found) + for idval in idlist: + idint, alias = _alias_or_int(idval) + for bugdict in r["bugs"]: + if idint and idint != bugdict.get("id", None): + continue + aliaslist = listify(bugdict.get("alias", None) or []) + if alias and alias not in aliaslist: + continue + ret.append(bugdict) + break return ret def _getbug(self, objid, **kwargs): @@ -1067,8 +1122,10 @@ class Bugzilla(object): def getbug(self, objid, include_fields=None, exclude_fields=None, extra_fields=None): - '''Return a Bug object with the full complement of bug data - already loaded.''' + """ + Return a Bug object with the full complement of bug data + already loaded. + """ data = self._getbug(objid, include_fields=include_fields, exclude_fields=exclude_fields, extra_fields=extra_fields) @@ -1077,9 +1134,11 @@ class Bugzilla(object): def getbugs(self, idlist, include_fields=None, exclude_fields=None, extra_fields=None, permissive=True): - '''Return a list of Bug objects with the full complement of bug data + """ + Return a list of Bug objects with the full complement of bug data already loaded. If there's a problem getting the data for a given id, - the corresponding item in the returned list will be None.''' + the corresponding item in the returned list will be None. + """ data = self._getbugs(idlist, include_fields=include_fields, exclude_fields=exclude_fields, extra_fields=extra_fields, permissive=permissive) @@ -1088,9 +1147,11 @@ class Bugzilla(object): for b in data] def get_comments(self, idlist): - '''Returns a dictionary of bugs and comments. The comments key will - be empty. See bugzilla docs for details''' - return self._proxy.Bug.comments({'ids': idlist}) + """ + Returns a dictionary of bugs and comments. The comments key will + be empty. See bugzilla docs for details + """ + return self._backend.bug_comments(idlist, {}) ################# @@ -1123,13 +1184,11 @@ class Bugzilla(object): alias=None, qa_whiteboard=None, devel_whiteboard=None, - boolean_query=None, bug_severity=None, priority=None, target_release=None, target_milestone=None, emailtype=None, - booleantype=None, include_fields=None, quicksearch=None, savedsearch=None, @@ -1137,12 +1196,13 @@ class Bugzilla(object): sub_component=None, tags=None, exclude_fields=None, - extra_fields=None): + extra_fields=None, + limit=None): """ Build a query string from passed arguments. Will handle query parameter differences between various bugzilla versions. - Most of the parameters should be self explanatory. However + Most of the parameters should be self-explanatory. However, if you want to perform a complex query, and easy way is to create it with the bugzilla web UI, copy the entire URL it generates, and pass it to the static method @@ -1154,15 +1214,10 @@ class Bugzilla(object): For details about the specific argument formats, see the bugzilla docs: https://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#search-bugs """ - if boolean_query or booleantype: - raise RuntimeError("boolean_query format is no longer supported. " - "If you need complicated URL queries, look into " - "query --from-url/url_to_query().") - query = { "alias": alias, - "product": self._listify(product), - "component": self._listify(component), + "product": listify(product), + "component": listify(component), "version": version, "id": bug_id, "short_desc": short_desc, @@ -1171,17 +1226,18 @@ class Bugzilla(object): "priority": priority, "target_release": target_release, "target_milestone": target_milestone, - "tag": self._listify(tags), + "tag": listify(tags), "quicksearch": quicksearch, "savedsearch": savedsearch, "sharer_id": savedsearch_sharer_id, + "limit": limit, # RH extensions... don't add any more. See comment below - "sub_components": self._listify(sub_component), + "sub_components": listify(sub_component), } def add_bool(bzkey, value, bool_id, booltype=None): - value = self._listify(value) + value = listify(value) if value is None: return bool_id @@ -1252,49 +1308,55 @@ class Bugzilla(object): return query def query(self, query): - '''Query bugzilla and return a list of matching bugs. + """ + Query bugzilla and return a list of matching bugs. query must be a dict with fields like those in in querydata['fields']. Returns a list of Bug objects. Also see the _query() method for details about the underlying implementation. - ''' - log.debug("Calling Bug.search with: %s", query) + """ try: - r = self._proxy.Bug.search(query) - except Fault as e: - + r = self._backend.bug_search(query) + log.debug("bug_search returned:\n%s", str(r)) + except Exception as e: # Try to give a hint in the error message if url_to_query # isn't supported by this bugzilla instance if ("query_format" not in str(e) or - "RHBugzilla" in str(e.__class__) or - self._check_version(5, 0)): + not BugzillaError.get_bugzilla_error_code(e) or + self._get_version() >= 5.0): raise raise BugzillaError("%s\nYour bugzilla instance does not " "appear to support API queries derived from bugzilla " - "web URL queries." % e) + "web URL queries." % e) from None log.debug("Query returned %s bugs", len(r['bugs'])) return [Bug(self, dict=b, autorefresh=self.bug_autorefresh) for b in r['bugs']] def pre_translation(self, query): - '''In order to keep the API the same, Bugzilla4 needs to process the + """ + In order to keep the API the same, Bugzilla4 needs to process the query and the result. This also applies to the refresh() function - ''' - pass + """ + if self._is_redhat_bugzilla: + _RHBugzillaConverters.pre_translation(query) + query.update(self._process_include_fields( + query.get("include_fields", []), None, None)) def post_translation(self, query, bug): - '''In order to keep the API the same, Bugzilla4 needs to process the + """ + In order to keep the API the same, Bugzilla4 needs to process the query and the result. This also applies to the refresh() function - ''' - pass + """ + if self._is_redhat_bugzilla: + _RHBugzillaConverters.post_translation(query, bug) def bugs_history_raw(self, bug_ids): - ''' + """ Experimental. Gets the history of changes for particular bugs in the database. - ''' - return self._proxy.Bug.history({'ids': bug_ids}) + """ + return self._backend.bug_history(bug_ids, {}) ####################################### @@ -1312,28 +1374,23 @@ class Bugzilla(object): build_update(), otherwise we cannot guarantee back compatibility. """ tmp = updates.copy() - tmp["ids"] = self._listify(ids) - - log.debug("Calling Bug.update with: %s", tmp) - return self._proxy.Bug.update(tmp) + return self._backend.bug_update(listify(ids), tmp) def update_tags(self, idlist, tags_add=None, tags_remove=None): - ''' + """ Updates the 'tags' field for a bug. - ''' + """ tags = {} if tags_add: - tags["add"] = self._listify(tags_add) + tags["add"] = listify(tags_add) if tags_remove: - tags["remove"] = self._listify(tags_remove) + tags["remove"] = listify(tags_remove) d = { - "ids": self._listify(idlist), "tags": tags, } - log.debug("Calling Bug.update_tags with: %s", d) - return self._proxy.Bug.update_tags(d) + return self._backend.bug_update_tags(listify(idlist), d) def update_flags(self, idlist, flags): """ @@ -1392,7 +1449,8 @@ class Bugzilla(object): internal_whiteboard=None, sub_component=None, flags=None, - comment_tags=None): + comment_tags=None, + minor_update=None): """ Returns a python dict() with properly formatted parameters to pass to update_bugs(). See bugzilla documentation for the format @@ -1401,18 +1459,28 @@ class Bugzilla(object): https://bugzilla.readthedocs.io/en/latest/api/core/v1/bug.html#create-bug """ ret = {} + rhbzret = {} # These are only supported for rhbugzilla - for key, val in [ - ("fixed_in", fixed_in), - ("devel_whiteboard", devel_whiteboard), - ("qa_whiteboard", qa_whiteboard), - ("internal_whiteboard", internal_whiteboard), - ("sub_component", sub_component), - ]: - if val is not None: - raise ValueError("bugzilla instance does not support " - "updating '%s'" % key) + # + # This should not be extended any more. + # If people want to handle custom fields, manually extend the + # returned dictionary. + rhbzargs = { + "fixed_in": fixed_in, + "devel_whiteboard": devel_whiteboard, + "qa_whiteboard": qa_whiteboard, + "internal_whiteboard": internal_whiteboard, + "sub_component": sub_component, + } + if self._is_redhat_bugzilla: + rhbzret = _RHBugzillaConverters.convert_build_update( + component=component, **rhbzargs) + else: + for key, val in rhbzargs.items(): + if val is not None: + raise ValueError("bugzilla instance does not support " + "updating '%s'" % key) def s(key, val, convert=None): if val is None: @@ -1426,7 +1494,7 @@ class Bugzilla(object): return def c(val): - val = self._listify(val) + val = listify(val) if convert: val = [convert(v) for v in val] return val @@ -1468,7 +1536,8 @@ class Bugzilla(object): s("whiteboard", whiteboard) s("work_time", work_time, float) s("flags", flags) - s("comment_tags", comment_tags, self._listify) + s("comment_tags", comment_tags, listify) + s("minor_update", minor_update, bool) add_dict("blocks", blocks_add, blocks_remove, blocks_set, convert=int) @@ -1484,6 +1553,7 @@ class Bugzilla(object): if comment_private: ret["comment"]["is_private"] = comment_private + ret.update(rhbzret) return ret @@ -1491,14 +1561,8 @@ class Bugzilla(object): # Methods for working with attachments # ######################################## - def _attachment_uri(self, attachid): - '''Returns the URI for the given attachment ID.''' - att_uri = self.url.replace('xmlrpc.cgi', 'attachment.cgi') - att_uri = att_uri + '?id=%s' % attachid - return att_uri - def attachfile(self, idlist, attachfile, description, **kwargs): - ''' + """ Attach a file to the given bug IDs. Returns the ID of the attachment or raises XMLRPC Fault if something goes wrong. @@ -1522,7 +1586,7 @@ class Bugzilla(object): Returns the list of attachment ids that were added. If only one attachment was added, we return the single int ID for back compat - ''' + """ if isinstance(attachfile, str): f = open(attachfile, "rb") elif hasattr(attachfile, 'read'): @@ -1543,21 +1607,20 @@ class Bugzilla(object): kwargs['summary'] = description data = f.read() - if not isinstance(data, bytes): + if not isinstance(data, bytes): # pragma: no cover data = data.encode(locale.getpreferredencoding()) - kwargs['data'] = Binary(data) - - kwargs['ids'] = self._listify(idlist) if 'file_name' not in kwargs and hasattr(f, "name"): kwargs['file_name'] = os.path.basename(f.name) if 'content_type' not in kwargs: - ctype = _detect_filetype(getattr(f, "name", None)) - if not ctype: - ctype = 'application/octet-stream' - kwargs['content_type'] = ctype + ctype = None + if kwargs['file_name']: + ctype = mimetypes.guess_type( + kwargs['file_name'], strict=False)[0] + kwargs['content_type'] = ctype or 'application/octet-stream' - ret = self._proxy.Bug.add_attachment(kwargs) + ret = self._backend.bug_attachment_create( + listify(idlist), data, kwargs) if "attachments" in ret: # Up to BZ 4.2 @@ -1570,37 +1633,52 @@ class Bugzilla(object): ret = ret[0] return ret + def openattachment_data(self, attachment_dict): + """ + Helper for turning passed API attachment dictionary into a + filelike object + """ + ret = BytesIO() + data = attachment_dict["data"] - def openattachment(self, attachid): - '''Get the contents of the attachment with the given attachment ID. - Returns a file-like object.''' - attachments = self.get_attachments(None, attachid) - data = attachments["attachments"][str(attachid)] - xmlrpcbinary = data["data"] + if hasattr(data, "data"): + # This is for xmlrpc Binary + content = data.data # pragma: no cover + else: + import base64 + content = base64.b64decode(data) - ret = BytesIO() - ret.write(xmlrpcbinary.data) - ret.name = data["file_name"] + ret.write(content) + ret.name = attachment_dict["file_name"] ret.seek(0) return ret + def openattachment(self, attachid): + """ + Get the contents of the attachment with the given attachment ID. + Returns a file-like object. + """ + attachments = self.get_attachments(None, attachid) + data = attachments["attachments"][str(attachid)] + return self.openattachment_data(data) + def updateattachmentflags(self, bugid, attachid, flagname, **kwargs): - ''' + """ Updates a flag for the given attachment ID. Optional keyword args are: status: new status for the flag ('-', '+', '?', 'X') requestee: new requestee for the flag - ''' + """ # Bug ID was used for the original custom redhat API, no longer # needed though ignore = bugid flags = {"name": flagname} flags.update(kwargs) - update = {'ids': [int(attachid)], 'flags': [flags]} + attachment_ids = [int(attachid)] + update = {'flags': [flags]} - log.debug("Calling Bug.update_attachment(%s)", update) - return self._proxy.Bug.update_attachment(update) + return self._backend.bug_attachment_update(attachment_ids, update) def get_attachments(self, ids, attachment_ids, include_fields=None, exclude_fields=None): @@ -1612,17 +1690,15 @@ class Bugzilla(object): https://bugzilla.readthedocs.io/en/latest/api/core/v1/attachment.html#get-attachment """ - params = { - "ids": self._listify(ids) or [], - "attachment_ids": self._listify(attachment_ids) or [], - } + params = {} if include_fields: - params["include_fields"] = self._listify(include_fields) + params["include_fields"] = listify(include_fields) if exclude_fields: - params["exclude_fields"] = self._listify(exclude_fields) + params["exclude_fields"] = listify(exclude_fields) - log.debug("Calling Bug.attachments(%s)", params) - return self._proxy.Bug.attachments(params) + if attachment_ids: + return self._backend.bug_attachment_get(attachment_ids, params) + return self._backend.bug_attachment_get_all(ids, params) ##################### @@ -1658,7 +1734,7 @@ class Bugzilla(object): sub_component=None, alias=None, comment_tags=None): - """" + """ Returns a python dict() with properly formatted parameters to pass to createbug(). See bugzilla documentation for the format of the individual fields: @@ -1668,15 +1744,15 @@ class Bugzilla(object): localdict = {} if blocks: - localdict["blocks"] = self._listify(blocks) + localdict["blocks"] = listify(blocks) if cc: - localdict["cc"] = self._listify(cc) + localdict["cc"] = listify(cc) if depends_on: - localdict["depends_on"] = self._listify(depends_on) + localdict["depends_on"] = listify(depends_on) if groups: - localdict["groups"] = self._listify(groups) + localdict["groups"] = listify(groups) if keywords: - localdict["keywords"] = self._listify(keywords) + localdict["keywords"] = listify(keywords) if description: localdict["description"] = description if comment_private: @@ -1700,14 +1776,15 @@ class Bugzilla(object): # Previous API required users specifying keyword args that mapped # to the XMLRPC arg names. Maintain that bad compat, but also allow # receiving a single dictionary like query() does - if kwargs and args: + if kwargs and args: # pragma: no cover raise BugzillaError("createbug: cannot specify positional " "args=%s with kwargs=%s, must be one or the " "other." % (args, kwargs)) if args: if len(args) > 1 or not isinstance(args[0], dict): - raise BugzillaError("createbug: positional arguments only " - "accept a single dictionary.") + raise BugzillaError( # pragma: no cover + "createbug: positional arguments only " + "accept a single dictionary.") data = args[0] else: data = kwargs @@ -1727,15 +1804,14 @@ class Bugzilla(object): return data def createbug(self, *args, **kwargs): - ''' + """ Create a bug with the given info. Returns a new Bug object. Check bugzilla API documentation for valid values, at least product, component, summary, version, and description need to be passed. - ''' + """ data = self._validate_createbug(*args, **kwargs) - log.debug("Calling Bug.create with: %s", data) - rawbug = self._proxy.Bug.create(data) + rawbug = self._backend.bug_create(data) return Bug(self, bug_id=rawbug["id"], autorefresh=self.bug_autorefresh) @@ -1744,54 +1820,28 @@ class Bugzilla(object): # Methods for handling Users # ############################## - def _getusers(self, ids=None, names=None, match=None): - '''Return a list of users that match criteria. - - :kwarg ids: list of user ids to return data on - :kwarg names: list of user names to return data on - :kwarg match: list of patterns. Returns users whose real name or - login name match the pattern. - :raises XMLRPC Fault: Code 51: if a Bad Login Name was sent to the - names array. - Code 304: if the user was not authorized to see user they - requested. - Code 505: user is logged out and can't use the match or ids - parameter. - - Available in Bugzilla-3.4+ - ''' - params = {} - if ids: - params['ids'] = self._listify(ids) - if names: - params['names'] = self._listify(names) - if match: - params['match'] = self._listify(match) - if not params: - raise BugzillaError('_get() needs one of ids, ' - ' names, or match kwarg.') - - log.debug("Calling User.get with: %s", params) - return self._proxy.User.get(params) - def getuser(self, username): - '''Return a bugzilla User for the given username + """ + Return a bugzilla User for the given username :arg username: The username used in bugzilla. :raises XMLRPC Fault: Code 51 if the username does not exist :returns: User record for the username - ''' + """ ret = self.getusers(username) return ret and ret[0] def getusers(self, userlist): - '''Return a list of Users from . + """ + Return a list of Users from . :userlist: List of usernames to lookup :returns: List of User records - ''' + """ + userlist = listify(userlist) + rawusers = self._backend.user_get({"names": userlist}) userobjs = [User(self, **rawuser) for rawuser in - self._getusers(names=userlist).get('users', [])] + rawusers.get('users', [])] # Return users in same order they were passed in ret = [] @@ -1806,16 +1856,19 @@ class Bugzilla(object): def searchusers(self, pattern): - '''Return a bugzilla User for the given list of patterns + """ + Return a bugzilla User for the given list of patterns :arg pattern: List of patterns to match against. :returns: List of User records - ''' + """ + rawusers = self._backend.user_get({"match": listify(pattern)}) return [User(self, **rawuser) for rawuser in - self._getusers(match=pattern).get('users', [])] + rawusers.get('users', [])] def createuser(self, email, name='', password=''): - '''Return a bugzilla User for the given username + """ + Return a bugzilla User for the given username :arg email: The email address to use in bugzilla :kwarg name: Real name to associate with the account @@ -1825,12 +1878,17 @@ class Bugzilla(object): Code 502 if the password is too short Code 503 if the password is too long :return: User record for the username - ''' - self._proxy.User.create(email, name, password) + """ + args = {"email": email} + if name: + args["name"] = name + if password: + args["password"] = password + self._backend.user_create(args) return self.getuser(email) def updateperms(self, user, action, groups): - ''' + """ A method to update the permissions (group membership) of a bugzilla user. @@ -1838,19 +1896,211 @@ class Bugzilla(object): also be a list of emails. :arg action: add, remove, or set :arg groups: list of groups to be added to (i.e. ['fedora_contrib']) - ''' - groups = self._listify(groups) + """ + groups = listify(groups) if action == "rem": action = "remove" if action not in ["add", "remove", "set"]: raise BugzillaError("Unknown user permission action '%s'" % action) update = { - "names": self._listify(user), + "names": listify(user), "groups": { action: groups, } } - log.debug("Call User.update with: %s", update) - return self._proxy.User.update(update) + return self._backend.user_update(update) + + + ############################### + # Methods for handling Groups # + ############################### + + def _getgroups(self, names, membership=False): + """ + Return a list of groups that match criteria. + + :kwarg ids: list of group ids to return data on + :kwarg membership: boolean specifying wether to query the members + of the group or not. + :raises XMLRPC Fault: Code 51: if a Bad Login Name was sent to the + names array. + Code 304: if the user was not authorized to see user they + requested. + Code 505: user is logged out and can't use the match or ids + parameter. + Code 805: logged in user do not have enough priviledges to view + groups. + """ + params = {"membership": membership} + params['names'] = listify(names) + return self._backend.group_get(params) + + def getgroup(self, name, membership=False): + """ + Return a bugzilla Group for the given name + + :arg name: The group name used in bugzilla. + :raises XMLRPC Fault: Code 51 if the name does not exist + :raises XMLRPC Fault: Code 805 if the user does not have enough + permissions to view groups + :returns: Group record for the name + """ + ret = self.getgroups(name, membership=membership) + return ret and ret[0] + + def getgroups(self, grouplist, membership=False): + """ + Return a list of Groups from . + + :userlist: List of group names to lookup + :returns: List of Group records + """ + grouplist = listify(grouplist) + groupobjs = [ + Group(self, **rawgroup) + for rawgroup in self._getgroups( + names=grouplist, membership=membership).get('groups', []) + ] + + # Return in same order they were passed in + ret = [] + for g in grouplist: + for gobj in groupobjs[:]: + if gobj.name == g: + groupobjs.remove(gobj) + ret.append(gobj) + break + ret += groupobjs + return ret + + + ############################# + # ExternalBugs API wrappers # + ############################# + + def add_external_tracker(self, bug_ids, ext_bz_bug_id, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_status=None, ext_description=None, + ext_priority=None): + """ + Wrapper method to allow adding of external tracking bugs using the + ExternalBugs::WebService::add_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#add-external-bug + + bug_ids: A single bug id or list of bug ids to have external trackers + added. + ext_bz_bug_id: The external bug id (ie: the bug number in the + external tracker). + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_status: The status of the external bug. + ext_description: The description of the external bug. + ext_priority: The priority of the external bug. + """ + param_dict = {'ext_bz_bug_id': ext_bz_bug_id} + if ext_type_id is not None: + param_dict['ext_type_id'] = ext_type_id + if ext_type_description is not None: + param_dict['ext_type_description'] = ext_type_description + if ext_type_url is not None: + param_dict['ext_type_url'] = ext_type_url + if ext_status is not None: + param_dict['ext_status'] = ext_status + if ext_description is not None: + param_dict['ext_description'] = ext_description + if ext_priority is not None: + param_dict['ext_priority'] = ext_priority + params = { + 'bug_ids': listify(bug_ids), + 'external_bugs': [param_dict], + } + return self._backend.externalbugs_add(params) + + def update_external_tracker(self, ids=None, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_bz_bug_id=None, bug_ids=None, + ext_status=None, ext_description=None, + ext_priority=None): + """ + Wrapper method to allow adding of external tracking bugs using the + ExternalBugs::WebService::update_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#update-external-bug + + ids: A single external tracker bug id or list of external tracker bug + ids. + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_bz_bug_id: A single external bug id or list of external bug ids + (ie: the bug number in the external tracker). + bug_ids: A single bug id or list of bug ids to have external tracker + info updated. + ext_status: The status of the external bug. + ext_description: The description of the external bug. + ext_priority: The priority of the external bug. + """ + params = {} + if ids is not None: + params['ids'] = listify(ids) + if ext_type_id is not None: + params['ext_type_id'] = ext_type_id + if ext_type_description is not None: + params['ext_type_description'] = ext_type_description + if ext_type_url is not None: + params['ext_type_url'] = ext_type_url + if ext_bz_bug_id is not None: + params['ext_bz_bug_id'] = listify(ext_bz_bug_id) + if bug_ids is not None: + params['bug_ids'] = listify(bug_ids) + if ext_status is not None: + params['ext_status'] = ext_status + if ext_description is not None: + params['ext_description'] = ext_description + if ext_priority is not None: + params['ext_priority'] = ext_priority + return self._backend.externalbugs_update(params) + + def remove_external_tracker(self, ids=None, ext_type_id=None, + ext_type_description=None, ext_type_url=None, + ext_bz_bug_id=None, bug_ids=None): + """ + Wrapper method to allow removal of external tracking bugs using the + ExternalBugs::WebService::remove_external_bug method. + + This is documented at + https://bugzilla.redhat.com/docs/en/html/integrating/api/Bugzilla/Extension/ExternalBugs/WebService.html#remove-external-bug + + ids: A single external tracker bug id or list of external tracker bug + ids. + ext_type_id: The external tracker id as used by Bugzilla. + ext_type_description: The external tracker description as used by + Bugzilla. + ext_type_url: The external tracker url as used by Bugzilla. + ext_bz_bug_id: A single external bug id or list of external bug ids + (ie: the bug number in the external tracker). + bug_ids: A single bug id or list of bug ids to have external tracker + info updated. + """ + params = {} + if ids is not None: + params['ids'] = listify(ids) + if ext_type_id is not None: + params['ext_type_id'] = ext_type_id + if ext_type_description is not None: + params['ext_type_description'] = ext_type_description + if ext_type_url is not None: + params['ext_type_url'] = ext_type_url + if ext_bz_bug_id is not None: + params['ext_bz_bug_id'] = listify(ext_bz_bug_id) + if bug_ids is not None: + params['bug_ids'] = listify(bug_ids) + return self._backend.externalbugs_remove(params) diff --git a/scripts/bugzilla/bug.py b/scripts/bugzilla/bug.py index e586e7f..f5c5f80 100644 --- a/scripts/bugzilla/bug.py +++ b/scripts/bugzilla/bug.py @@ -1,71 +1,65 @@ -# base.py - the base classes etc. for a Python interface to bugzilla -# # Copyright (C) 2007, 2008, 2009, 2010 Red Hat Inc. # Author: Will Woods # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - -from __future__ import unicode_literals -import locale +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +import copy from logging import getLogger -import sys + log = getLogger(__name__) class Bug(object): - '''A container object for a bug report. Requires a Bugzilla instance - + """ + A container object for a bug report. Requires a Bugzilla instance - every Bug is on a Bugzilla, obviously. Optional keyword args: dict=DICT - populate attributes with the result of a getBug() call bug_id=ID - if dict does not contain bug_id, this is required before you can read any attributes or make modifications to this bug. - ''' + """ def __init__(self, bugzilla, bug_id=None, dict=None, autorefresh=False): # pylint: disable=redefined-builtin # API had pre-existing issue that we can't change ('dict' usage) self.bugzilla = bugzilla - self._bug_fields = [] + self._rawdata = {} self.autorefresh = autorefresh + # pylint: disable=protected-access + self._aliases = self.bugzilla._get_bug_aliases() + # pylint: enable=protected-access + if not dict: dict = {} if bug_id: dict["id"] = bug_id - log.debug("Bug(%s)", sorted(dict.keys())) self._update_dict(dict) - self.weburl = bugzilla.url.replace('xmlrpc.cgi', 'show_bug.cgi?id=%i' % self.bug_id) def __str__(self): - '''Return a simple string representation of this bug - - This is available only for compatibility. Using 'str(bug)' and - 'print(bug)' is not recommended because of potential encoding issues. - Please use unicode(bug) where possible. - ''' - if sys.version_info[0] >= 3: - return self.__unicode__() - else: - return self.__unicode__().encode( - locale.getpreferredencoding(), 'replace') + """ + Return a simple string representation of this bug + """ + return self.__unicode__() def __unicode__(self): - '''Return a simple unicode string representation of this bug''' + """ + Return a simple unicode string representation of this bug + """ return "#%-6s %-10s - %s - %s" % (self.bug_id, self.bug_status, self.assigned_to, self.summary) def __repr__(self): - return '' % (self.bug_id, self.bugzilla.url, - id(self)) + url = "" + if self.bugzilla: + url = self.bugzilla.url + return '' % (self.bug_id, url, id(self)) def __getattr__(self, name): refreshed = False @@ -75,11 +69,7 @@ class Bug(object): # have never been called. return self.__dict__[name] - # pylint: disable=protected-access - aliases = self.bugzilla._get_bug_aliases() - # pylint: enable=protected-access - - for newname, oldname in aliases: + for newname, oldname in self._aliases: if name == oldname and newname in self.__dict__: return self.__dict__[newname] @@ -110,47 +100,52 @@ class Bug(object): "to adjust your include_fields for getbug/query." % name) raise AttributeError(msg) + def get_raw_data(self): + """ + Return the raw API dictionary data that has been used to + populate this bug + """ + return copy.deepcopy(self._rawdata) + def refresh(self, include_fields=None, exclude_fields=None, extra_fields=None): - ''' + """ Refresh the bug with the latest data from bugzilla - ''' + """ # pylint: disable=protected-access + extra_fields = list(self._rawdata.keys()) + (extra_fields or []) r = self.bugzilla._getbug(self.bug_id, include_fields=include_fields, exclude_fields=exclude_fields, - extra_fields=self._bug_fields + (extra_fields or [])) + extra_fields=extra_fields) # pylint: enable=protected-access self._update_dict(r) reload = refresh - def _update_dict(self, newdict): - ''' - Update internal dictionary, in a way that ensures no duplicate - entries are stored WRT field aliases - ''' + def _translate_dict(self, newdict): if self.bugzilla: self.bugzilla.post_translation({}, newdict) - # pylint: disable=protected-access - aliases = self.bugzilla._get_bug_aliases() - # pylint: enable=protected-access - - for newname, oldname in aliases: - if oldname not in newdict: - continue - - if newname not in newdict: - newdict[newname] = newdict[oldname] - elif newdict[newname] != newdict[oldname]: - log.debug("Update dict contained differing alias values " - "d[%s]=%s and d[%s]=%s , dropping the value " - "d[%s]", newname, newdict[newname], oldname, - newdict[oldname], oldname) - del(newdict[oldname]) - - for key in newdict.keys(): - if key not in self._bug_fields: - self._bug_fields.append(key) + for newname, oldname in self._aliases: + if oldname not in newdict: + continue + + if newname not in newdict: + newdict[newname] = newdict[oldname] + elif newdict[newname] != newdict[oldname]: + log.debug("Update dict contained differing alias values " + "d[%s]=%s and d[%s]=%s , dropping the value " + "d[%s]", newname, newdict[newname], oldname, + newdict[oldname], oldname) + del(newdict[oldname]) + + + def _update_dict(self, newdict): + """ + Update internal dictionary, in a way that ensures no duplicate + entries are stored WRT field aliases + """ + self._translate_dict(newdict) + self._rawdata.update(newdict) self.__dict__.update(newdict) if 'id' not in self.__dict__ and 'bug_id' not in self.__dict__: @@ -162,14 +157,15 @@ class Bug(object): ################## def __getstate__(self): - ret = {} - for key in self._bug_fields: - ret[key] = self.__dict__[key] + ret = self._rawdata.copy() + ret["_aliases"] = self._aliases return ret def __setstate__(self, vals): - self._bug_fields = [] + self._rawdata = {} self.bugzilla = None + self._aliases = vals.get("_aliases", []) + self.autorefresh = False self._update_dict(vals) @@ -178,12 +174,12 @@ class Bug(object): ##################### def setstatus(self, status, comment=None, private=False): - ''' + """ Update the status for this bug report. Commonly-used values are ASSIGNED, MODIFIED, and NEEDINFO. To change bugs to RESOLVED, use .close() instead. - ''' + """ # Note: fedora bodhi uses this function vals = self.bugzilla.build_update(status=status, comment=comment, @@ -194,7 +190,8 @@ class Bug(object): def close(self, resolution, dupeid=None, fixedin=None, comment=None, isprivate=False): - '''Close this bug. + """ + Close this bug. Valid values for resolution are in bz.querydefaults['resolution_list'] For bugzilla.redhat.com that's: ['NOTABUG', 'WONTFIX', 'DEFERRED', 'WORKSFORME', 'CURRENTRELEASE', @@ -206,14 +203,14 @@ class Bug(object): version that fixes the bug. You can optionally add a comment while closing the bug. Set 'isprivate' to True if you want that comment to be private. - ''' + """ # Note: fedora bodhi uses this function vals = self.bugzilla.build_update(comment=comment, comment_private=isprivate, resolution=resolution, dupe_of=dupeid, fixed_in=fixedin, - status="RESOLVED") + status=str("RESOLVED")) log.debug("close: update=%s", vals) return self.bugzilla.update_bugs(self.bug_id, vals) @@ -225,7 +222,7 @@ class Bug(object): def setassignee(self, assigned_to=None, qa_contact=None, comment=None): - ''' + """ Set any of the assigned_to or qa_contact fields to a new bugzilla account, with an optional comment, e.g. setassignee(assigned_to='wwoods@redhat.com') @@ -235,7 +232,7 @@ class Bug(object): will throw a ValueError. Returns [bug_id, mailresults]. - ''' + """ if not (assigned_to or qa_contact): raise ValueError("You must set one of assigned_to " " or qa_contact") @@ -248,11 +245,11 @@ class Bug(object): return self.bugzilla.update_bugs(self.bug_id, vals) def addcc(self, cclist, comment=None): - ''' + """ Adds the given email addresses to the CC list for this bug. cclist: list of email addresses (strings) comment: optional comment to add to the bug - ''' + """ vals = self.bugzilla.build_update(comment=comment, cc_add=cclist) log.debug("addcc: update=%s", vals) @@ -260,9 +257,9 @@ class Bug(object): return self.bugzilla.update_bugs(self.bug_id, vals) def deletecc(self, cclist, comment=None): - ''' + """ Removes the given email addresses from the CC list for this bug. - ''' + """ vals = self.bugzilla.build_update(comment=comment, cc_remove=cclist) log.debug("deletecc: update=%s", vals) @@ -275,10 +272,10 @@ class Bug(object): #################### def addcomment(self, comment, private=False): - ''' + """ Add the given comment to this bug. Set private to True to mark this comment as private. - ''' + """ # Note: fedora bodhi uses this function vals = self.bugzilla.build_update(comment=comment, comment_private=private) @@ -287,9 +284,9 @@ class Bug(object): return self.bugzilla.update_bugs(self.bug_id, vals) def getcomments(self): - ''' + """ Returns an array of comment dictionaries for this bug - ''' + """ comment_list = self.bugzilla.get_comments([self.bug_id]) return comment_list['bugs'][str(self.bug_id)]['comments'] @@ -377,18 +374,19 @@ class Bug(object): return [a["id"] for a in self.get_attachments(exclude_fields=["data"])] def get_history_raw(self): - ''' + """ Experimental. Get the history of changes for this bug. - ''' + """ return self.bugzilla.bugs_history_raw([self.bug_id]) class User(object): - '''Container object for a bugzilla User. + """ + Container object for a bugzilla User. :arg bugzilla: Bugzilla instance that this User belongs to. Rest of the params come straight from User.get() - ''' + """ def __init__(self, bugzilla, **kwargs): self.bugzilla = bugzilla self.__userid = kwargs.get('id') @@ -440,11 +438,74 @@ class User(object): self.__dict__.update(newuser.__dict__) def updateperms(self, action, groups): - ''' + """ A method to update the permissions (group membership) of a bugzilla user. :arg action: add, remove, or set :arg groups: list of groups to be added to (i.e. ['fedora_contrib']) - ''' + """ self.bugzilla.updateperms(self.name, action, groups) + + +class Group(object): + """ + Container object for a bugzilla Group. + + :arg bugzilla: Bugzilla instance that this Group belongs to. + Rest of the params come straight from Group.get() + """ + def __init__(self, bugzilla, **kwargs): + self.bugzilla = bugzilla + self.__groupid = kwargs.get('id') + + self.name = kwargs.get('name') + self.description = kwargs.get('description', self.name) + self.is_active = kwargs.get('is_active', False) + self.icon_url = kwargs.get('icon_url', None) + self.is_active_bug_group = kwargs.get('is_active_bug_group', None) + + self.membership = kwargs.get('membership', []) + self.__member_emails = set() + self._refresh_member_emails_list() + + ######################## + # Read-only attributes # + ######################## + + # We make these properties so that the user cannot set them. They are + # unaffected by the update() method so it would be misleading to let them + # be changed. + @property + def groupid(self): + return self.__groupid + + @property + def member_emails(self): + return sorted(self.__member_emails) + + def _refresh_member_emails_list(self): + """ + Refresh the list of emails of the members of the group. + """ + if self.membership: + for m in self.membership: + if "email" in m: + self.__member_emails.add(m["email"]) + + def refresh(self, membership=False): + """ + Update Group object with latest info from bugzilla + """ + newgroup = self.bugzilla.getgroup( + self.name, membership=membership) + self.__dict__.update(newgroup.__dict__) + self._refresh_member_emails_list() + + def members(self): + """ + Retrieve the members of this Group from bugzilla + """ + if not self.membership: + self.refresh(membership=True) + return self.membership diff --git a/scripts/bugzilla/exceptions.py b/scripts/bugzilla/exceptions.py new file mode 100644 index 0000000..d884df0 --- /dev/null +++ b/scripts/bugzilla/exceptions.py @@ -0,0 +1,38 @@ +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + + +class BugzillaError(Exception): + """ + Error raised in the Bugzilla client code. + """ + @staticmethod + def get_bugzilla_error_string(exc): + """ + Helper to return the bugzilla instance error message from an + XMLRPC Fault, or any other exception type that's raised from bugzilla + interaction + """ + return getattr(exc, "faultString", str(exc)) + + @staticmethod + def get_bugzilla_error_code(exc): + """ + Helper to return the bugzilla instance error code from an + XMLRPC Fault, or any other exception type that's raised from bugzilla + interaction + """ + for propname in ["faultCode", "code"]: + if hasattr(exc, propname): + return getattr(exc, propname) + return None + + def __init__(self, message, code=None): + """ + :param code: The error code from the remote bugzilla instance. Only + set if the error came directly from the remove bugzilla + """ + self.code = code + if self.code: + message += " (code=%s)" % self.code + Exception.__init__(self, message) diff --git a/scripts/bugzilla/oldclasses.py b/scripts/bugzilla/oldclasses.py index 18169e7..e579fb9 100644 --- a/scripts/bugzilla/oldclasses.py +++ b/scripts/bugzilla/oldclasses.py @@ -1,23 +1,64 @@ -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. from .base import Bugzilla -from .rhbugzilla import RHBugzilla - # These are old compat classes. Nothing new should be added here, # and these should not be altered -class Bugzilla3(Bugzilla): pass -class Bugzilla32(Bugzilla): pass -class Bugzilla34(Bugzilla): pass -class Bugzilla36(Bugzilla): pass -class Bugzilla4(Bugzilla): pass -class Bugzilla42(Bugzilla): pass -class Bugzilla44(Bugzilla): pass -class NovellBugzilla(Bugzilla): pass -class RHBugzilla3(RHBugzilla): pass -class RHBugzilla4(RHBugzilla): pass + +class Bugzilla3(Bugzilla): + pass + + +class Bugzilla32(Bugzilla): + pass + + +class Bugzilla34(Bugzilla): + pass + + +class Bugzilla36(Bugzilla): + pass + + +class Bugzilla4(Bugzilla): + pass + + +class Bugzilla42(Bugzilla): + pass + + +class Bugzilla44(Bugzilla): + pass + + +class NovellBugzilla(Bugzilla): + pass + + +class RHBugzilla(Bugzilla): + """ + Helper class for historical bugzilla.redhat.com back compat + + Historically this class used many more non-upstream methods, but + in 2012 RH started dropping most of its custom bits. By that time, + upstream BZ had most of the important functionality. + + Much of the remaining code here is just trying to keep things operating + in python-bugzilla back compatible manner. + + This class was written using bugzilla.redhat.com's API docs: + https://bugzilla.redhat.com/docs/en/html/api/ + """ + _is_redhat_bugzilla = True + + +class RHBugzilla3(RHBugzilla): + pass + + +class RHBugzilla4(RHBugzilla): + pass diff --git a/scripts/bugzilla/rhbugzilla.py b/scripts/bugzilla/rhbugzilla.py index 55ee601..10b4594 100644 --- a/scripts/bugzilla/rhbugzilla.py +++ b/scripts/bugzilla/rhbugzilla.py @@ -1,352 +1,7 @@ -# rhbugzilla.py - a Python interface to Red Hat Bugzilla using xmlrpclib. -# -# Copyright (C) 2008-2012 Red Hat Inc. -# Author: Will Woods -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. +# This work is licensed under the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. -from logging import getLogger +# This class needs to live in rhbugzilla.py to preserve historical +# 'bugzilla.rhbugzilla' import compat -from .base import Bugzilla - -log = getLogger(__name__) - - -class RHBugzilla(Bugzilla): - ''' - Bugzilla class for connecting Red Hat's forked bugzilla instance, - bugzilla.redhat.com - - Historically this class used many more non-upstream methods, but - in 2012 RH started dropping most of its custom bits. By that time, - upstream BZ had most of the important functionality. - - Much of the remaining code here is just trying to keep things operating - in python-bugzilla back compatible manner. - - This class was written using bugzilla.redhat.com's API docs: - https://bugzilla.redhat.com/docs/en/html/api/ - ''' - def _init_class_state(self): - def _add_both_alias(newname, origname): - self._add_field_alias(newname, origname, is_api=False) - self._add_field_alias(origname, newname, is_bug=False) - - _add_both_alias('fixed_in', 'cf_fixed_in') - _add_both_alias('qa_whiteboard', 'cf_qa_whiteboard') - _add_both_alias('devel_whiteboard', 'cf_devel_whiteboard') - _add_both_alias('internal_whiteboard', 'cf_internal_whiteboard') - - self._add_field_alias('component', 'components', is_bug=False) - self._add_field_alias('version', 'versions', is_bug=False) - # Yes, sub_components is the field name the API expects - self._add_field_alias('sub_components', 'sub_component', is_bug=False) - - # flags format isn't exactly the same but it's the closest approx - self._add_field_alias('flags', 'flag_types') - - self._getbug_extra_fields = self._getbug_extra_fields + [ - "comments", "description", - "external_bugs", "flags", "sub_components", - "tags", - ] - self._supports_getbug_extra_fields = True - - - ###################### - # Bug update methods # - ###################### - - def build_update(self, **kwargs): - # pylint: disable=arguments-differ - adddict = {} - - def pop(key, destkey): - val = kwargs.pop(key, None) - if val is None: - return - adddict[destkey] = val - - def get_sub_component(): - val = kwargs.pop("sub_component", None) - if val is None: - return - - if not isinstance(val, dict): - component = self._listify(kwargs.get("component")) - if not component: - raise ValueError("component must be specified if " - "specifying sub_component") - val = {component[0]: val} - adddict["sub_components"] = val - - def get_alias(): - # RHBZ has a custom extension to allow a bug to have multiple - # aliases, so the format of aliases is - # {"add": [...], "remove": [...]} - # But that means in order to approximate upstream, behavior - # which just overwrites the existing alias, we need to read - # the bug's state first to know what string to remove. Which - # we can't do, since we don't know the bug numbers at this point. - # So fail for now. - # - # The API should provide {"set": [...]} - # https://bugzilla.redhat.com/show_bug.cgi?id=1173114 - # - # Implementation will go here when it's available - pass - - pop("fixed_in", "cf_fixed_in") - pop("qa_whiteboard", "cf_qa_whiteboard") - pop("devel_whiteboard", "cf_devel_whiteboard") - pop("internal_whiteboard", "cf_internal_whiteboard") - - get_sub_component() - get_alias() - - vals = Bugzilla.build_update(self, **kwargs) - vals.update(adddict) - - return vals - - def add_external_tracker(self, bug_ids, ext_bz_bug_id, ext_type_id=None, - ext_type_description=None, ext_type_url=None, - ext_status=None, ext_description=None, - ext_priority=None): - """ - Wrapper method to allow adding of external tracking bugs using the - ExternalBugs::WebService::add_external_bug method. - - This is documented at - https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#add_external_bug - - bug_ids: A single bug id or list of bug ids to have external trackers - added. - ext_bz_bug_id: The external bug id (ie: the bug number in the - external tracker). - ext_type_id: The external tracker id as used by Bugzilla. - ext_type_description: The external tracker description as used by - Bugzilla. - ext_type_url: The external tracker url as used by Bugzilla. - ext_status: The status of the external bug. - ext_description: The description of the external bug. - ext_priority: The priority of the external bug. - """ - param_dict = {'ext_bz_bug_id': ext_bz_bug_id} - if ext_type_id is not None: - param_dict['ext_type_id'] = ext_type_id - if ext_type_description is not None: - param_dict['ext_type_description'] = ext_type_description - if ext_type_url is not None: - param_dict['ext_type_url'] = ext_type_url - if ext_status is not None: - param_dict['ext_status'] = ext_status - if ext_description is not None: - param_dict['ext_description'] = ext_description - if ext_priority is not None: - param_dict['ext_priority'] = ext_priority - params = { - 'bug_ids': self._listify(bug_ids), - 'external_bugs': [param_dict], - } - - log.debug("Calling ExternalBugs.add_external_bug(%s)", params) - return self._proxy.ExternalBugs.add_external_bug(params) - - def update_external_tracker(self, ids=None, ext_type_id=None, - ext_type_description=None, ext_type_url=None, - ext_bz_bug_id=None, bug_ids=None, - ext_status=None, ext_description=None, - ext_priority=None): - """ - Wrapper method to allow adding of external tracking bugs using the - ExternalBugs::WebService::update_external_bug method. - - This is documented at - https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#update_external_bug - - ids: A single external tracker bug id or list of external tracker bug - ids. - ext_type_id: The external tracker id as used by Bugzilla. - ext_type_description: The external tracker description as used by - Bugzilla. - ext_type_url: The external tracker url as used by Bugzilla. - ext_bz_bug_id: A single external bug id or list of external bug ids - (ie: the bug number in the external tracker). - bug_ids: A single bug id or list of bug ids to have external tracker - info updated. - ext_status: The status of the external bug. - ext_description: The description of the external bug. - ext_priority: The priority of the external bug. - """ - params = {} - if ids is not None: - params['ids'] = self._listify(ids) - if ext_type_id is not None: - params['ext_type_id'] = ext_type_id - if ext_type_description is not None: - params['ext_type_description'] = ext_type_description - if ext_type_url is not None: - params['ext_type_url'] = ext_type_url - if ext_bz_bug_id is not None: - params['ext_bz_bug_id'] = self._listify(ext_bz_bug_id) - if bug_ids is not None: - params['bug_ids'] = self._listify(bug_ids) - if ext_status is not None: - params['ext_status'] = ext_status - if ext_description is not None: - params['ext_description'] = ext_description - if ext_priority is not None: - params['ext_priority'] = ext_priority - - log.debug("Calling ExternalBugs.update_external_bug(%s)", params) - return self._proxy.ExternalBugs.update_external_bug(params) - - def remove_external_tracker(self, ids=None, ext_type_id=None, - ext_type_description=None, ext_type_url=None, - ext_bz_bug_id=None, bug_ids=None): - """ - Wrapper method to allow removal of external tracking bugs using the - ExternalBugs::WebService::remove_external_bug method. - - This is documented at - https://bugzilla.redhat.com/docs/en/html/api/extensions/ExternalBugs/lib/WebService.html#remove_external_bug - - ids: A single external tracker bug id or list of external tracker bug - ids. - ext_type_id: The external tracker id as used by Bugzilla. - ext_type_description: The external tracker description as used by - Bugzilla. - ext_type_url: The external tracker url as used by Bugzilla. - ext_bz_bug_id: A single external bug id or list of external bug ids - (ie: the bug number in the external tracker). - bug_ids: A single bug id or list of bug ids to have external tracker - info updated. - """ - params = {} - if ids is not None: - params['ids'] = self._listify(ids) - if ext_type_id is not None: - params['ext_type_id'] = ext_type_id - if ext_type_description is not None: - params['ext_type_description'] = ext_type_description - if ext_type_url is not None: - params['ext_type_url'] = ext_type_url - if ext_bz_bug_id is not None: - params['ext_bz_bug_id'] = self._listify(ext_bz_bug_id) - if bug_ids is not None: - params['bug_ids'] = self._listify(bug_ids) - - log.debug("Calling ExternalBugs.remove_external_bug(%s)", params) - return self._proxy.ExternalBugs.remove_external_bug(params) - - - ################# - # Query methods # - ################# - - def pre_translation(self, query): - '''Translates the query for possible aliases''' - old = query.copy() - - if 'bug_id' in query: - if not isinstance(query['bug_id'], list): - query['id'] = query['bug_id'].split(',') - else: - query['id'] = query['bug_id'] - del query['bug_id'] - - if 'component' in query: - if not isinstance(query['component'], list): - query['component'] = query['component'].split(',') - - if 'include_fields' not in query and 'column_list' not in query: - return - - if 'include_fields' not in query: - query['include_fields'] = [] - if 'column_list' in query: - query['include_fields'] = query['column_list'] - del query['column_list'] - - # We need to do this for users here for users that - # don't call build_query - query.update(self._process_include_fields(query["include_fields"], - None, None)) - - if old != query: - log.debug("RHBugzilla pretranslated query to: %s", query) - - def post_translation(self, query, bug): - ''' - Convert the results of getbug back to the ancient RHBZ value - formats - ''' - ignore = query - - # RHBZ _still_ returns component and version as lists, which - # deviates from upstream. Copy the list values to components - # and versions respectively. - if 'component' in bug and "components" not in bug: - val = bug['component'] - bug['components'] = isinstance(val, list) and val or [val] - bug['component'] = bug['components'][0] - - if 'version' in bug and "versions" not in bug: - val = bug['version'] - bug['versions'] = isinstance(val, list) and val or [val] - bug['version'] = bug['versions'][0] - - # sub_components isn't too friendly of a format, add a simpler - # sub_component value - if 'sub_components' in bug and 'sub_component' not in bug: - val = bug['sub_components'] - bug['sub_component'] = "" - if isinstance(val, dict): - values = [] - for vallist in val.values(): - values += vallist - bug['sub_component'] = " ".join(values) - - def build_external_tracker_boolean_query(self, *args, **kwargs): - ignore1 = args - ignore2 = kwargs - raise RuntimeError("Building external boolean queries is " - "no longer supported. Please build a URL query " - "via the bugzilla web UI and pass it to 'query --from-url' " - "or url_to_query()") - - - def build_query(self, **kwargs): - # pylint: disable=arguments-differ - - # We previously accepted a text format to approximate boolean - # queries, and only for RHBugzilla. Upstream bz has --from-url - # support now, so point people to that instead so we don't have - # to document and maintain this logic anymore - def _warn_bool(kwkey): - vallist = self._listify(kwargs.get(kwkey, None)) - for value in vallist or []: - for s in value.split(" "): - if s not in ["|", "&", "!"]: - continue - log.warning("%s value '%s' appears to use the now " - "unsupported boolean formatting, your query may " - "be incorrect. If you need complicated URL queries, " - "look into bugzilla --from-url/url_to_query().", - kwkey, value) - return - - _warn_bool("fixed_in") - _warn_bool("blocked") - _warn_bool("dependson") - _warn_bool("flag") - _warn_bool("qa_whiteboard") - _warn_bool("devel_whiteboard") - _warn_bool("alias") - - return Bugzilla.build_query(self, **kwargs) +from .oldclasses import RHBugzilla # pylint: disable=unused-import diff --git a/scripts/git_sort/tests/Docker/opensuse-15.4.Dockerfile b/scripts/git_sort/tests/Docker/opensuse-15.4.Dockerfile new file mode 100644 index 0000000..1d5cbc7 --- /dev/null +++ b/scripts/git_sort/tests/Docker/opensuse-15.4.Dockerfile @@ -0,0 +1,24 @@ +# https://hub.docker.com/r/opensuse/leap/ +FROM opensuse/leap:15.4 AS base + +RUN zypper -n ref + +FROM base AS packages + +RUN zypper -n in git python3 python3-dbm rcs + +RUN git config --global user.email "you@example.com" +RUN git config --global user.name "Your Name" + +COPY Kernel.gpg /tmp +RUN rpmkeys --import /tmp/Kernel.gpg +RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_15_SP4/Kernel:tools.repo +RUN zypper -n in python3-pygit2 quilt + +FROM packages + +VOLUME /scripts + +WORKDIR /scripts/git_sort + +CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/Docker/opensuse-tumbleweed.Dockerfile b/scripts/git_sort/tests/Docker/opensuse-tumbleweed.Dockerfile new file mode 100644 index 0000000..af6cfbc --- /dev/null +++ b/scripts/git_sort/tests/Docker/opensuse-tumbleweed.Dockerfile @@ -0,0 +1,24 @@ +# https://hub.docker.com/r/opensuse/tumbleweed/ +FROM opensuse/tumbleweed AS base + +RUN zypper -n ref + +FROM base AS packages + +RUN zypper -n in git python3 python3-dbm python3-pygit2 rcs util-linux + +RUN git config --global user.email "you@example.com" +RUN git config --global user.name "Your Name" + +COPY Kernel.gpg /tmp +RUN rpmkeys --import /tmp/Kernel.gpg +RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/openSUSE_Factory/Kernel:tools.repo +RUN zypper -n in --from Kernel_tools quilt + +FROM packages + +VOLUME /scripts + +WORKDIR /scripts/git_sort + +CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/Docker/sle12-sp4.Dockerfile b/scripts/git_sort/tests/Docker/sle12-sp4.Dockerfile new file mode 100644 index 0000000..49d8f9c --- /dev/null +++ b/scripts/git_sort/tests/Docker/sle12-sp4.Dockerfile @@ -0,0 +1,33 @@ +# http://registry.suse.de/ +FROM registry.suse.de/suse/containers/sle-server/12-sp4/containers/suse/sles12sp4:latest AS base + +RUN rpm -e container-suseconnect +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/GA/standard/SUSE:SLE-12:GA.repo +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/Update/standard/SUSE:SLE-12:Update.repo +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD1/ DVD1 +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD2/ DVD2 +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD3/ DVD3 +# RUN zypper -n ar -G http://updates.suse.de/SUSE/Products/SLE-SDK/12-SP4/$(rpm -E %_arch)/product/ SDK +RUN zypper -n ar http://download.suse.de/update/build.suse.de/SUSE/Updates/SLE-SERVER/12-SP4/$(rpm -E %_arch)/update/SUSE:Updates:SLE-SERVER:12-SP4:$(rpm -E %_arch).repo + +RUN zypper -n ref + +FROM base AS packages + +RUN zypper -n in git-core python3 python3-dbm rcs + +RUN git config --global user.email "you@example.com" +RUN git config --global user.name "Your Name" + +COPY Kernel.gpg /tmp +RUN rpmkeys --import /tmp/Kernel.gpg +RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_12_SP4/Kernel:tools.repo +RUN zypper -n in python3-pygit2 quilt + +FROM packages + +VOLUME /scripts + +WORKDIR /scripts/git_sort + +CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/Docker/sle12-sp5.Dockerfile b/scripts/git_sort/tests/Docker/sle12-sp5.Dockerfile new file mode 100644 index 0000000..f5e2f58 --- /dev/null +++ b/scripts/git_sort/tests/Docker/sle12-sp5.Dockerfile @@ -0,0 +1,33 @@ +# http://registry.suse.de/ +FROM registry.suse.de/suse/containers/sle-server/12-sp5/containers/suse/sles12sp5:latest AS base + +RUN rpm -e container-suseconnect +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/GA/standard/SUSE:SLE-12:GA.repo +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/Update/standard/SUSE:SLE-12:Update.repo +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD1/ DVD1 +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD2/ DVD2 +RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD3/ DVD3 +# RUN zypper -n ar -G http://updates.suse.de/SUSE/Products/SLE-SDK/12-SP5/$(rpm -E %_arch)/product/ SDK +RUN zypper -n ar http://download.suse.de/update/build.suse.de/SUSE/Updates/SLE-SERVER/12-SP5/$(rpm -E %_arch)/update/SUSE:Updates:SLE-SERVER:12-SP5:$(rpm -E %_arch).repo + +RUN zypper -n ref + +FROM base AS packages + +RUN zypper -n in git-core python3 python3-dbm rcs + +RUN git config --global user.email "you@example.com" +RUN git config --global user.name "Your Name" + +COPY Kernel.gpg /tmp +RUN rpmkeys --import /tmp/Kernel.gpg +RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_12_SP5/Kernel:tools.repo +RUN zypper -n in python3-pygit2 quilt + +FROM packages + +VOLUME /scripts + +WORKDIR /scripts/git_sort + +CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/Docker/sle15.Dockerfile b/scripts/git_sort/tests/Docker/sle15.Dockerfile new file mode 100644 index 0000000..4d1cc54 --- /dev/null +++ b/scripts/git_sort/tests/Docker/sle15.Dockerfile @@ -0,0 +1,27 @@ +# http://registry.suse.de/ +FROM registry.suse.de/suse/sle-15/update/images/suse/sle15:latest AS base + +RUN rpm -e container-suseconnect +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-15:/GA/standard/SUSE:SLE-15:GA.repo +RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-15:/Update/standard/SUSE:SLE-15:Update.repo +RUN zypper -n ref + +FROM base AS packages + +RUN zypper -n in git-core python3 python3-dbm rcs awk + +RUN git config --global user.email "you@example.com" +RUN git config --global user.name "Your Name" + +COPY Kernel.gpg /tmp +RUN rpmkeys --import /tmp/Kernel.gpg +RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_15/Kernel:tools.repo +RUN zypper -n in python3-pygit2 quilt + +FROM packages + +VOLUME /scripts + +WORKDIR /scripts/git_sort + +CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/opensuse-15.4/Dockerfile b/scripts/git_sort/tests/opensuse-15.4/Dockerfile deleted file mode 100644 index 1d5cbc7..0000000 --- a/scripts/git_sort/tests/opensuse-15.4/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# https://hub.docker.com/r/opensuse/leap/ -FROM opensuse/leap:15.4 AS base - -RUN zypper -n ref - -FROM base AS packages - -RUN zypper -n in git python3 python3-dbm rcs - -RUN git config --global user.email "you@example.com" -RUN git config --global user.name "Your Name" - -COPY Kernel.gpg /tmp -RUN rpmkeys --import /tmp/Kernel.gpg -RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_15_SP4/Kernel:tools.repo -RUN zypper -n in python3-pygit2 quilt - -FROM packages - -VOLUME /scripts - -WORKDIR /scripts/git_sort - -CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/opensuse-tumbleweed/Dockerfile b/scripts/git_sort/tests/opensuse-tumbleweed/Dockerfile deleted file mode 100644 index af6cfbc..0000000 --- a/scripts/git_sort/tests/opensuse-tumbleweed/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# https://hub.docker.com/r/opensuse/tumbleweed/ -FROM opensuse/tumbleweed AS base - -RUN zypper -n ref - -FROM base AS packages - -RUN zypper -n in git python3 python3-dbm python3-pygit2 rcs util-linux - -RUN git config --global user.email "you@example.com" -RUN git config --global user.name "Your Name" - -COPY Kernel.gpg /tmp -RUN rpmkeys --import /tmp/Kernel.gpg -RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/openSUSE_Factory/Kernel:tools.repo -RUN zypper -n in --from Kernel_tools quilt - -FROM packages - -VOLUME /scripts - -WORKDIR /scripts/git_sort - -CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/run_all.sh b/scripts/git_sort/tests/run_all.sh index f63ab5b..e0109b1 100755 --- a/scripts/git_sort/tests/run_all.sh +++ b/scripts/git_sort/tests/run_all.sh @@ -1,6 +1,28 @@ -#!/bin/bash +#!/bin/sh -libdir=$(dirname "$(readlink -f "$0")") +enable_x() { + local enable=true + while [ $# -gt 0 ] ; do + { [ "$1" = "-q" ] || [ "$1" = "--quiet" ] ; } && enable=false + shift + done + $enable && set -x + } + +enable_x "$@" + +testdir=$(dirname "$(readlink -f "$0")") +keys="Kernel.gpg" + +for key in $keys ; do + cp -a $testdir/../../lib/SUSE/$key $testdir/Docker +done + +trap ' +for key in $keys ; do + rm $testdir/Docker/$key +done +' EXIT for release in \ sle12-sp4 \ @@ -10,14 +32,12 @@ for release in \ opensuse-tumbleweed \ ; do echo "Building container image for $release..." - cp -a $libdir/../../lib/SUSE/Kernel.gpg $libdir/$release - docker build -q -t gs-test-$release "$libdir/$release" + docker build "$@" -t gs-test-$release -f $testdir/Docker/$release.Dockerfile --build-arg release=$release $testdir/Docker ret=$? - rm -f $libdir/$release/Kernel.gpg [ $ret -eq 0 ] || exit $? echo "Running tests in $release:" docker run --rm --name=gs-test-$release \ - --mount type=bind,source="$libdir/../../",target=/scripts,readonly \ + --mount type=bind,source="$testdir/../../",target=/scripts,readonly \ gs-test-$release ret=$? [ $ret -eq 0 ] || exit $? diff --git a/scripts/git_sort/tests/sle12-sp4/Dockerfile b/scripts/git_sort/tests/sle12-sp4/Dockerfile deleted file mode 100644 index 49d8f9c..0000000 --- a/scripts/git_sort/tests/sle12-sp4/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# http://registry.suse.de/ -FROM registry.suse.de/suse/containers/sle-server/12-sp4/containers/suse/sles12sp4:latest AS base - -RUN rpm -e container-suseconnect -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/GA/standard/SUSE:SLE-12:GA.repo -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/Update/standard/SUSE:SLE-12:Update.repo -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD1/ DVD1 -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD2/ DVD2 -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP4-Server-GM/$(rpm -E %_arch)/DVD3/ DVD3 -# RUN zypper -n ar -G http://updates.suse.de/SUSE/Products/SLE-SDK/12-SP4/$(rpm -E %_arch)/product/ SDK -RUN zypper -n ar http://download.suse.de/update/build.suse.de/SUSE/Updates/SLE-SERVER/12-SP4/$(rpm -E %_arch)/update/SUSE:Updates:SLE-SERVER:12-SP4:$(rpm -E %_arch).repo - -RUN zypper -n ref - -FROM base AS packages - -RUN zypper -n in git-core python3 python3-dbm rcs - -RUN git config --global user.email "you@example.com" -RUN git config --global user.name "Your Name" - -COPY Kernel.gpg /tmp -RUN rpmkeys --import /tmp/Kernel.gpg -RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_12_SP4/Kernel:tools.repo -RUN zypper -n in python3-pygit2 quilt - -FROM packages - -VOLUME /scripts - -WORKDIR /scripts/git_sort - -CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/sle12-sp5/Dockerfile b/scripts/git_sort/tests/sle12-sp5/Dockerfile deleted file mode 100644 index f5e2f58..0000000 --- a/scripts/git_sort/tests/sle12-sp5/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# http://registry.suse.de/ -FROM registry.suse.de/suse/containers/sle-server/12-sp5/containers/suse/sles12sp5:latest AS base - -RUN rpm -e container-suseconnect -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/GA/standard/SUSE:SLE-12:GA.repo -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-12:/Update/standard/SUSE:SLE-12:Update.repo -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD1/ DVD1 -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD2/ DVD2 -RUN zypper -n ar http://download.suse.de/install/SLP/SLE-12-SP5-Server-GM/$(rpm -E %_arch)/DVD3/ DVD3 -# RUN zypper -n ar -G http://updates.suse.de/SUSE/Products/SLE-SDK/12-SP5/$(rpm -E %_arch)/product/ SDK -RUN zypper -n ar http://download.suse.de/update/build.suse.de/SUSE/Updates/SLE-SERVER/12-SP5/$(rpm -E %_arch)/update/SUSE:Updates:SLE-SERVER:12-SP5:$(rpm -E %_arch).repo - -RUN zypper -n ref - -FROM base AS packages - -RUN zypper -n in git-core python3 python3-dbm rcs - -RUN git config --global user.email "you@example.com" -RUN git config --global user.name "Your Name" - -COPY Kernel.gpg /tmp -RUN rpmkeys --import /tmp/Kernel.gpg -RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_12_SP5/Kernel:tools.repo -RUN zypper -n in python3-pygit2 quilt - -FROM packages - -VOLUME /scripts - -WORKDIR /scripts/git_sort - -CMD python3 -m unittest discover -v diff --git a/scripts/git_sort/tests/sle15/Dockerfile b/scripts/git_sort/tests/sle15/Dockerfile deleted file mode 100644 index 002074f..0000000 --- a/scripts/git_sort/tests/sle15/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# http://registry.suse.de/ -FROM registry.suse.de/suse/containers/sle-server/15/containers/bci/python:3.6 AS base - -RUN rpm -e container-suseconnect -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-15:/GA/standard/SUSE:SLE-15:GA.repo -RUN zypper -n ar http://download.suse.de/ibs/SUSE:/SLE-15:/Update/standard/SUSE:SLE-15:Update.repo -RUN zypper -n ref - -FROM base AS packages - -RUN zypper -n in git-core python3 python3-dbm rcs awk - -RUN git config --global user.email "you@example.com" -RUN git config --global user.name "Your Name" - -COPY Kernel.gpg /tmp -RUN rpmkeys --import /tmp/Kernel.gpg -RUN zypper -n ar https://download.opensuse.org/repositories/Kernel:/tools/SLE_15/Kernel:tools.repo -RUN zypper -n in python3-pygit2 quilt - -FROM packages - -VOLUME /scripts - -WORKDIR /scripts/git_sort - -CMD python3 -m unittest discover -v diff --git a/scripts/run_oldconfig.sh b/scripts/run_oldconfig.sh index 357692a..d18aa1f 100755 --- a/scripts/run_oldconfig.sh +++ b/scripts/run_oldconfig.sh @@ -254,6 +254,9 @@ scripts="${prefix}scripts" if test -e "${prefix}rpm/config.sh"; then source "$_" fi +if [ "$VARIANT" = "-vanilla" ] ; then + VANILLA_ONLY=1 +fi if test -z "$set_flavor" && test "$VANILLA_ONLY" = 1 -o -e .is_vanilla; then set_flavor=vanilla fi diff --git a/scripts/tar-up.sh b/scripts/tar-up.sh index 2ea9a1a..01f5d03 100755 --- a/scripts/tar-up.sh +++ b/scripts/tar-up.sh @@ -191,7 +191,7 @@ CLEANFILES=() trap 'if test -n "$CLEANFILES"; then rm -rf "${CLEANFILES[@]}"; fi' EXIT tmpdir=$(mktemp -dt ${0##*/}.XXXXXX) CLEANFILES=("${CLEANFILES[@]}" "$tmpdir") -rpmfiles=$(ls rpm/* | grep -v "~$") +rpmfiles=$(ls -d rpm/* | grep -v -e "~$" -e "[.]orig$" -e "[.]rej$" | { while read x ; do [ -d "$x" ] || echo "$x" ; done ; } ) rpmstatus=$(for i in $rpmfiles ; do git status -s $i ; done) [ -z "$rpmstatus" ] || { inconsistent=true ; echo "$rpmstatus" ; }