From ebbdf9bd5a0f7b94307edef08bd1ff3102c5304d Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: May 04 2025 17:57:19 +0000 Subject: Factor out runner calls from pillar Runner execution causes pillar rendering to slow down significantly. Move hosts functions to regular execution module and call those instead (this works as execution module calls in the pillar are executed on the master as well). Example rendering of the pillar for witch1.i.o.o yielded total times of: - ~11 seconds before this patch - ~3 seconds after this patch Comparing the profiling logs showed specificially the pillar templates with runner calls to have contributed to the poor first result. The role.monitoring.master pillar still executes a runner due to a dependency on the Salt mine, this will require separate refactoring. This however only affects a single minion, all other minions will already benefit from the improvements here. Reference: https://github.com/saltstack/salt/issues/43163 Signed-off-by: Georg Pfuetzenreuter --- diff --git a/bin/prepare_test_env.sh b/bin/prepare_test_env.sh index 523368a..bac8da8 100755 --- a/bin/prepare_test_env.sh +++ b/bin/prepare_test_env.sh @@ -150,7 +150,6 @@ tee /etc/salt/minion.d/roots.conf >/dev/null <<-EOF ln -s "$PWD/salt" /srv/salt salt-call --local saltutil.runner saltutil.sync_modules -salt-call --local saltutil.runner saltutil.sync_runners salt-call --local saltutil.sync_grains salt-call --local saltutil.sync_modules salt-call --local saltutil.sync_states diff --git a/pillar/cluster/atlas/services.sls b/pillar/cluster/atlas/services.sls index db57b01..b793c85 100644 --- a/pillar/cluster/atlas/services.sls +++ b/pillar/cluster/atlas/services.sls @@ -10,7 +10,7 @@ haproxy: 'matrix', ] %} - - src_limit_exclude src {{ salt['saltutil.runner']('os_pillar.get_host_ip6', arg=[host, True]) }}/128 + - src_limit_exclude src {{ salt['os_pillar.get_host_ip6'](host, True) }}/128 {%- endfor %} - no_x-frame-option var(txn.host) -m str chat.opensuse.org - no_x-frame-option var(txn.host) -m str dimension.opensuse.org diff --git a/pillar/common/hosts.sls b/pillar/common/hosts.sls index c6fc695..1ac71d2 100644 --- a/pillar/common/hosts.sls +++ b/pillar/common/hosts.sls @@ -3,8 +3,8 @@ {%- set id_host = id.split('.')[0] -%} {#- try hosts.yaml lookup -#} -{%- set address6 = salt['saltutil.runner']('os_pillar.get_host_ip6', arg=[id_host, True]) -%} -{%- set address4 = salt['saltutil.runner']('os_pillar.get_host_ip4', arg=[id_host, True]) -%} +{%- set address6 = salt['os_pillar.get_host_ip6'](id_host, True) -%} +{%- set address4 = salt['os_pillar.get_host_ip4'](id_host, True) -%} {#- if not successful, try grains lookup -#} {#- IPv4, based on private addresses -#} diff --git a/pillar/common/init.sls b/pillar/common/init.sls index db86ac7..1d42874 100644 --- a/pillar/common/init.sls +++ b/pillar/common/init.sls @@ -4,7 +4,7 @@ {% set osrelease = salt['grains.get']('osrelease') %} {%- set virtual = salt['grains.get']('virtual') -%} {%- set site = salt['grains.get']('site') -%} -{%- set address = salt['saltutil.runner']('os_pillar.get_host_ip6', arg=[grains['host'], True]) -%} +{%- set address = salt['os_pillar.get_host_ip6'](grains['host'], True) -%} {%- set configure_ntp = salt['grains.get']('configure_ntp', True) %} {%- set id = grains['id'] %} diff --git a/pillar/common/monitoring/listen.sls b/pillar/common/monitoring/listen.sls index 6783804..158b761 100644 --- a/pillar/common/monitoring/listen.sls +++ b/pillar/common/monitoring/listen.sls @@ -4,7 +4,7 @@ This pillar template implements logic to determine the best suitable listening a {#- first try: query hosts.yaml #} -{%- set address = salt['saltutil.runner']('os_pillar.get_host_ip6', arg=[grains['host'], True]) -%} +{%- set address = salt['os_pillar.get_host_ip6'](grains['host'], True) -%} {#- second try: iterate over FQDN derived addresses and find one in the PRG2 supernet where we can assume machines have only a single relevant IPv6 equipped interface #} diff --git a/salt/_modules/os_pillar.py b/salt/_modules/os_pillar.py new file mode 100644 index 0000000..d3db711 --- /dev/null +++ b/salt/_modules/os_pillar.py @@ -0,0 +1,112 @@ +""" +Execution module for fetching host configuraton data from hosts.yaml + +Author: Georg Pfuetzenreuter + +Copyright (C) 2023-2025 openSUSE contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import yaml + +repository = '/srv/salt-git' +file_root = f'{repository}/salt' +pillar_root = f'{repository}/pillar' + + +def _load(dataset): + with open(f'{pillar_root}/infra/{dataset}.yaml') as fh: + return yaml.safe_load(fh) + + +def _load_hosts(): + return _load('hosts') + + +def _load_networks(): + return _load('networks') + + +def get_host(host): + hosts = _load_hosts() + return hosts.get(host, {}) + + +def get_host_interfaces(host): + return get_host(host).get('interfaces', {}) + + +def get_host_primary_interface(host): + config = get_host(host) + interfaces = config.get('interfaces', {}) + + if 'primary_interface' in config: + primary_interface = config['primary_interface'] + elif len(interfaces) > 0: + primary_interface = next(iter(interfaces)) + else: + primary_interface = 'eth0' + + return interfaces.get(primary_interface, {}) + + +def get_host_disks(host): + return get_host(host).get('disks', {}) + + +def get_host_ips(host): + config = get_host_primary_interface(host) + result = {'ip4': None, 'ip6': None, 'pseudo_ip4': None} + result['ip4'] = config.get('ip4') + result['ip6'] = config.get('ip6') + result['pseudo_ip4'] = config.get('pseudo_ip4') + + return result + + +def get_host_ip4(host, strip_cidr=False): + address = get_host_ips(host).get('ip4') + if strip_cidr and address is not None: + address = __salt__['os_network.strip_cidr'](address) # noqa F821 + return address + + +def get_host_ip6(host, strip_cidr=False): + address = get_host_ips(host).get('ip6') + if strip_cidr and address is not None: + address = __salt__['os_network.strip_cidr'](address) # noqa F821 + return address + + +def get_host_pseudo_ip4(host): + config = get_host_ips(host) + return config.get('pseudo_ip4') + + +def get_network(site, name): + config = _load_networks() + return config.get(site, {}).get(name, {}) + + +def get_host_ip4to6(host, prefix=None, network=None): + host_config = get_host_ip4(host) + if host_config is not None: + if prefix is None: + if network is None: + network = 'openSUSE-NAT46-Pool' + prefix = get_network('pseudo', network).get('net6') + if prefix is not None: + return __salt__['os_network.convert_4to6'](__salt__['os_network.strip_cidr'](host_config), prefix) # noqa F821 + return None diff --git a/salt/_runners/os_pillar.py b/salt/_runners/os_pillar.py deleted file mode 100644 index 9c063a3..0000000 --- a/salt/_runners/os_pillar.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Extension module for fetching host configuraton data from hosts.yaml - -Author: Georg Pfuetzenreuter - -Copyright (C) 2023 openSUSE contributors - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import yaml - -repository = '/srv/salt-git' -file_root = f'{repository}/salt' -pillar_root = f'{repository}/pillar' - - -def _load(dataset): - with open(f'{pillar_root}/infra/{dataset}.yaml') as fh: - return yaml.safe_load(fh) - - -def _load_hosts(): - return _load('hosts') - - -def _load_networks(): - return _load('networks') - - -def get_host(host): - hosts = _load_hosts() - return hosts.get(host, {}) - - -def get_host_interfaces(host): - return get_host(host).get('interfaces', {}) - - -def get_host_primary_interface(host): - config = get_host(host) - interfaces = config.get('interfaces', {}) - - if 'primary_interface' in config: - primary_interface = config['primary_interface'] - elif len(interfaces) > 0: - primary_interface = next(iter(interfaces)) - else: - primary_interface = 'eth0' - - return interfaces.get(primary_interface, {}) - - -def get_host_disks(host): - return get_host(host).get('disks', {}) - - -def get_host_ips(host): - config = get_host_primary_interface(host) - result = {'ip4': None, 'ip6': None, 'pseudo_ip4': None} - result['ip4'] = config.get('ip4') - result['ip6'] = config.get('ip6') - result['pseudo_ip4'] = config.get('pseudo_ip4') - - return result - - -def get_host_ip4(host, strip_cidr=False): - address = get_host_ips(host).get('ip4') - if strip_cidr and address is not None: - address = __salt__['salt.cmd']('os_network.strip_cidr', address) # noqa F821 - return address - - -def get_host_ip6(host, strip_cidr=False): - address = get_host_ips(host).get('ip6') - if strip_cidr and address is not None: - address = __salt__['salt.cmd']('os_network.strip_cidr', address) # noqa F821 - return address - - -def get_host_pseudo_ip4(host): - config = get_host_ips(host) - return config.get('pseudo_ip4') - - -def get_network(site, name): - config = _load_networks() - return config.get(site, {}).get(name, {}) - - -def get_host_ip4to6(host, prefix=None, network=None): - host_config = get_host_ip4(host) - if host_config is not None: - if prefix is None: - if network is None: - network = 'openSUSE-NAT46-Pool' - prefix = get_network('pseudo', network).get('net6') - if prefix is not None: - return __salt__['salt.cmd']('os_network.convert_4to6', __salt__['salt.cmd']('os_network.strip_cidr', host_config), prefix) # noqa F821 - return None