diff options
Diffstat (limited to 'contrib/inventory/foreman.py')
| -rwxr-xr-x | contrib/inventory/foreman.py | 662 |
1 files changed, 0 insertions, 662 deletions
diff --git a/contrib/inventory/foreman.py b/contrib/inventory/foreman.py deleted file mode 100755 index 343cf26c9d..0000000000 --- a/contrib/inventory/foreman.py +++ /dev/null @@ -1,662 +0,0 @@ -#!/usr/bin/env python -# vim: set fileencoding=utf-8 : -# -# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>, -# Daniel Lobato Garcia <dlobatog@redhat.com> -# -# This script is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with it. If not, see <http://www.gnu.org/licenses/>. -# -# This is somewhat based on cobbler inventory - -# Stdlib imports -# __future__ imports must occur at the beginning of file -from __future__ import print_function -import json -import argparse -import copy -import os -import re -import sys -from time import time, sleep -from collections import defaultdict -from distutils.version import LooseVersion, StrictVersion - -# 3rd party imports -import requests -if LooseVersion(requests.__version__) < LooseVersion('1.1.0'): - print('This script requires python-requests 1.1 as a minimum version') - sys.exit(1) - -from requests.auth import HTTPBasicAuth - -from ansible.module_utils._text import to_text -from ansible.module_utils.six.moves import configparser as ConfigParser - - -def json_format_dict(data, pretty=False): - """Converts a dict to a JSON object and dumps it as a formatted string""" - - if pretty: - return json.dumps(data, sort_keys=True, indent=2) - else: - return json.dumps(data) - - -class ForemanInventory(object): - - def __init__(self): - self.inventory = defaultdict(list) # A list of groups and the hosts in that group - self.cache = dict() # Details about hosts in the inventory - self.params = dict() # Params of each host - self.facts = dict() # Facts of each host - self.hostgroups = dict() # host groups - self.hostcollections = dict() # host collections - self.session = None # Requests session - self.config_paths = [ - "/etc/ansible/foreman.ini", - os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini', - ] - env_value = os.environ.get('FOREMAN_INI_PATH') - if env_value is not None: - self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) - - def read_settings(self): - """Reads the settings from the foreman.ini file""" - - config = ConfigParser.SafeConfigParser() - config.read(self.config_paths) - - # Foreman API related - try: - self.foreman_url = config.get('foreman', 'url') - self.foreman_user = config.get('foreman', 'user') - self.foreman_pw = config.get('foreman', 'password', raw=True) - self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: - print("Error parsing configuration: %s" % e, file=sys.stderr) - return False - - # Inventory Report Related - try: - self.foreman_use_reports_api = config.getboolean('foreman', 'use_reports_api') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.foreman_use_reports_api = True - - try: - self.want_organization = config.getboolean('report', 'want_organization') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_organization = True - - try: - self.want_location = config.getboolean('report', 'want_location') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_location = True - - try: - self.want_IPv4 = config.getboolean('report', 'want_ipv4') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_IPv4 = True - - try: - self.want_IPv6 = config.getboolean('report', 'want_ipv6') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_IPv6 = False - - try: - self.want_host_group = config.getboolean('report', 'want_host_group') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_host_group = True - - try: - self.want_host_params = config.getboolean('report', 'want_host_params') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_host_params = False - - try: - self.want_subnet = config.getboolean('report', 'want_subnet') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_subnet = True - - try: - self.want_subnet_v6 = config.getboolean('report', 'want_subnet_v6') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_subnet_v6 = False - - try: - self.want_smart_proxies = config.getboolean('report', 'want_smart_proxies') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_smart_proxies = True - - try: - self.want_content_facet_attributes = config.getboolean('report', 'want_content_facet_attributes') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_content_facet_attributes = False - - try: - self.report_want_facts = config.getboolean('report', 'want_facts') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.report_want_facts = True - - try: - self.poll_interval = config.getint('report', 'poll_interval') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.poll_interval = 10 - - # Ansible related - try: - group_patterns = config.get('ansible', 'group_patterns') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - group_patterns = "[]" - - self.group_patterns = json.loads(group_patterns) - - try: - self.group_prefix = config.get('ansible', 'group_prefix') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.group_prefix = "foreman_" - - try: - self.want_facts = config.getboolean('ansible', 'want_facts') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_facts = True - - self.want_facts = self.want_facts and self.report_want_facts - - try: - self.want_hostcollections = config.getboolean('ansible', 'want_hostcollections') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_hostcollections = False - - try: - self.want_ansible_ssh_host = config.getboolean('ansible', 'want_ansible_ssh_host') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.want_ansible_ssh_host = False - - # Do we want parameters to be interpreted if possible as JSON? (no by default) - try: - self.rich_params = config.getboolean('ansible', 'rich_params') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.rich_params = False - - try: - self.host_filters = config.get('foreman', 'host_filters') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.host_filters = None - - # Cache related - try: - cache_path = os.path.expanduser(config.get('cache', 'path')) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - cache_path = '.' - (script, ext) = os.path.splitext(os.path.basename(__file__)) - self.cache_path_cache = cache_path + "/%s.cache" % script - self.cache_path_inventory = cache_path + "/%s.index" % script - self.cache_path_params = cache_path + "/%s.params" % script - self.cache_path_facts = cache_path + "/%s.facts" % script - self.cache_path_hostcollections = cache_path + "/%s.hostcollections" % script - try: - self.cache_max_age = config.getint('cache', 'max_age') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.cache_max_age = 60 - try: - self.scan_new_hosts = config.getboolean('cache', 'scan_new_hosts') - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.scan_new_hosts = False - - return True - - def parse_cli_args(self): - """Command line argument processing""" - - parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on foreman') - parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)') - parser.add_argument('--host', action='store', help='Get all the variables about a specific instance') - parser.add_argument('--refresh-cache', action='store_true', default=False, - help='Force refresh of cache by making API requests to foreman (default: False - use cache files)') - self.args = parser.parse_args() - - def _get_session(self): - if not self.session: - self.session = requests.session() - self.session.auth = HTTPBasicAuth(self.foreman_user, self.foreman_pw) - self.session.verify = self.foreman_ssl_verify - return self.session - - def _get_json(self, url, ignore_errors=None, params=None): - if params is None: - params = {} - params['per_page'] = 250 - - page = 1 - results = [] - s = self._get_session() - while True: - params['page'] = page - ret = s.get(url, params=params) - if ignore_errors and ret.status_code in ignore_errors: - break - ret.raise_for_status() - json = ret.json() - # /hosts/:id has not results key - if 'results' not in json: - return json - # Facts are returned as dict in results not list - if isinstance(json['results'], dict): - return json['results'] - # List of all hosts is returned paginaged - results = results + json['results'] - if len(results) >= json['subtotal']: - break - page += 1 - if len(json['results']) == 0: - print("Did not make any progress during loop. " - "expected %d got %d" % (json['total'], len(results)), - file=sys.stderr) - break - return results - - def _use_inventory_report(self): - if not self.foreman_use_reports_api: - return False - status_url = "%s/api/v2/status" % self.foreman_url - result = self._get_json(status_url) - foreman_version = (LooseVersion(result.get('version')) >= LooseVersion('1.24.0')) - return foreman_version - - def _fetch_params(self): - options, params = ("no", "yes"), dict() - params["Organization"] = options[self.want_organization] - params["Location"] = options[self.want_location] - params["IPv4"] = options[self.want_IPv4] - params["IPv6"] = options[self.want_IPv6] - params["Facts"] = options[self.want_facts] - params["Host Group"] = options[self.want_host_group] - params["Host Collections"] = options[self.want_hostcollections] - params["Subnet"] = options[self.want_subnet] - params["Subnet v6"] = options[self.want_subnet_v6] - params["Smart Proxies"] = options[self.want_smart_proxies] - params["Content Attributes"] = options[self.want_content_facet_attributes] - params["Host Parameters"] = options[self.want_host_params] - if self.host_filters: - params["Hosts"] = self.host_filters - return params - - def _post_request(self): - url = "%s/ansible/api/v2/ansible_inventories/schedule" % self.foreman_url - session = self._get_session() - params = {'input_values': self._fetch_params()} - ret = session.post(url, json=params) - if not ret: - raise Exception("Error scheduling inventory report on foreman. Please check foreman logs!") - url = "{0}/{1}".format(self.foreman_url, ret.json().get('data_url')) - response = session.get(url) - while response: - if response.status_code != 204: - break - else: - sleep(self.poll_interval) - response = session.get(url) - if not response: - raise Exception("Error receiving inventory report from foreman. Please check foreman logs!") - else: - return response.json() - - def _get_hosts(self): - url = "%s/api/v2/hosts" % self.foreman_url - - params = {} - if self.host_filters: - params['search'] = self.host_filters - - return self._get_json(url, params=params) - - def _get_host_data_by_id(self, hid): - url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid) - return self._get_json(url) - - def _get_facts_by_id(self, hid): - url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid) - return self._get_json(url) - - def _resolve_params(self, host_params): - """Convert host params to dict""" - params = {} - - for param in host_params: - name = param['name'] - if self.rich_params: - try: - params[name] = json.loads(param['value']) - except ValueError: - params[name] = param['value'] - else: - params[name] = param['value'] - - return params - - def _get_facts(self, host): - """Fetch all host facts of the host""" - if not self.want_facts: - return {} - - ret = self._get_facts_by_id(host['id']) - if len(ret.values()) == 0: - facts = {} - elif len(ret.values()) == 1: - facts = list(ret.values())[0] - else: - raise ValueError("More than one set of facts returned for '%s'" % host) - return facts - - def write_to_cache(self, data, filename): - """Write data in JSON format to a file""" - json_data = json_format_dict(data, True) - cache = open(filename, 'w') - cache.write(json_data) - cache.close() - - def _write_cache(self): - self.write_to_cache(self.cache, self.cache_path_cache) - self.write_to_cache(self.inventory, self.cache_path_inventory) - self.write_to_cache(self.params, self.cache_path_params) - self.write_to_cache(self.facts, self.cache_path_facts) - self.write_to_cache(self.hostcollections, self.cache_path_hostcollections) - - def to_safe(self, word): - '''Converts 'bad' characters in a string to underscores - so they can be used as Ansible groups - - >>> ForemanInventory.to_safe("foo-bar baz") - 'foo_barbaz' - ''' - regex = r"[^A-Za-z0-9\_]" - return re.sub(regex, "_", word.replace(" ", "")) - - def update_cache(self, scan_only_new_hosts=False): - """Make calls to foreman and save the output in a cache""" - use_inventory_report = self._use_inventory_report() - if use_inventory_report: - self._update_cache_inventory(scan_only_new_hosts) - else: - self._update_cache_host_api(scan_only_new_hosts) - - def _update_cache_inventory(self, scan_only_new_hosts): - self.groups = dict() - self.hosts = dict() - try: - inventory_report_response = self._post_request() - except Exception: - self._update_cache_host_api(scan_only_new_hosts) - return - host_data = json.loads(inventory_report_response) - for host in host_data: - if not(host) or (host["name"] in self.cache.keys() and scan_only_new_hosts): - continue - dns_name = host['name'] - - host_params = host.pop('host_parameters', {}) - fact_list = host.pop('facts', {}) - content_facet_attributes = host.get('content_attributes', {}) or {} - - # Create ansible groups for hostgroup - group = 'host_group' - val = host.get(group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - # Create ansible groups for environment, location and organization - for group in ['environment', 'location', 'organization']: - val = host.get('%s' % group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - for group in ['lifecycle_environment', 'content_view']: - val = content_facet_attributes.get('%s_name' % group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - params = host_params - - # Ansible groups by parameters in host groups and Foreman host - # attributes. - groupby = dict() - for k, v in params.items(): - groupby[k] = self.to_safe(to_text(v)) - - # The name of the ansible groups is given by group_patterns: - for pattern in self.group_patterns: - try: - key = pattern.format(**groupby) - self.inventory[key].append(dns_name) - except KeyError: - pass # Host not part of this group - - if self.want_hostcollections: - hostcollections = host.get('host_collections') - - if hostcollections: - # Create Ansible groups for host collections - for hostcollection in hostcollections: - safe_key = self.to_safe('%shostcollection_%s' % (self.group_prefix, hostcollection.lower())) - self.inventory[safe_key].append(dns_name) - - self.hostcollections[dns_name] = hostcollections - - self.cache[dns_name] = host - self.params[dns_name] = params - self.facts[dns_name] = fact_list - self.inventory['all'].append(dns_name) - self._write_cache() - - def _update_cache_host_api(self, scan_only_new_hosts): - """Make calls to foreman and save the output in a cache""" - - self.groups = dict() - self.hosts = dict() - - for host in self._get_hosts(): - if host['name'] in self.cache.keys() and scan_only_new_hosts: - continue - dns_name = host['name'] - - host_data = self._get_host_data_by_id(host['id']) - host_params = host_data.get('all_parameters', {}) - - # Create ansible groups for hostgroup - group = 'hostgroup' - val = host.get('%s_title' % group) or host.get('%s_name' % group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - # Create ansible groups for environment, location and organization - for group in ['environment', 'location', 'organization']: - val = host.get('%s_name' % group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - for group in ['lifecycle_environment', 'content_view']: - val = host.get('content_facet_attributes', {}).get('%s_name' % group) - if val: - safe_key = self.to_safe('%s%s_%s' % ( - to_text(self.group_prefix), - group, - to_text(val).lower() - )) - self.inventory[safe_key].append(dns_name) - - params = self._resolve_params(host_params) - - # Ansible groups by parameters in host groups and Foreman host - # attributes. - groupby = dict() - for k, v in params.items(): - groupby[k] = self.to_safe(to_text(v)) - - # The name of the ansible groups is given by group_patterns: - for pattern in self.group_patterns: - try: - key = pattern.format(**groupby) - self.inventory[key].append(dns_name) - except KeyError: - pass # Host not part of this group - - if self.want_hostcollections: - hostcollections = host_data.get('host_collections') - - if hostcollections: - # Create Ansible groups for host collections - for hostcollection in hostcollections: - safe_key = self.to_safe('%shostcollection_%s' % (self.group_prefix, hostcollection['name'].lower())) - self.inventory[safe_key].append(dns_name) - - self.hostcollections[dns_name] = hostcollections - - self.cache[dns_name] = host - self.params[dns_name] = params - self.facts[dns_name] = self._get_facts(host) - self.inventory['all'].append(dns_name) - self._write_cache() - - def is_cache_valid(self): - """Determines if the cache is still valid""" - if os.path.isfile(self.cache_path_cache): - mod_time = os.path.getmtime(self.cache_path_cache) - current_time = time() - if (mod_time + self.cache_max_age) > current_time: - if (os.path.isfile(self.cache_path_inventory) and - os.path.isfile(self.cache_path_params) and - os.path.isfile(self.cache_path_facts)): - return True - return False - - def load_inventory_from_cache(self): - """Read the index from the cache file sets self.index""" - - with open(self.cache_path_inventory, 'r') as fp: - self.inventory = json.load(fp) - - def load_params_from_cache(self): - """Read the index from the cache file sets self.index""" - - with open(self.cache_path_params, 'r') as fp: - self.params = json.load(fp) - - def load_facts_from_cache(self): - """Read the index from the cache file sets self.facts""" - - if not self.want_facts: - return - with open(self.cache_path_facts, 'r') as fp: - self.facts = json.load(fp) - - def load_hostcollections_from_cache(self): - """Read the index from the cache file sets self.hostcollections""" - - if not self.want_hostcollections: - return - with open(self.cache_path_hostcollections, 'r') as fp: - self.hostcollections = json.load(fp) - - def load_cache_from_cache(self): - """Read the cache from the cache file sets self.cache""" - - with open(self.cache_path_cache, 'r') as fp: - self.cache = json.load(fp) - - def get_inventory(self): - if self.args.refresh_cache or not self.is_cache_valid(): - self.update_cache() - else: - self.load_inventory_from_cache() - self.load_params_from_cache() - self.load_facts_from_cache() - self.load_hostcollections_from_cache() - self.load_cache_from_cache() - if self.scan_new_hosts: - self.update_cache(True) - - def get_host_info(self): - """Get variables about a specific host""" - - if not self.cache or len(self.cache) == 0: - # Need to load index from cache - self.load_cache_from_cache() - - if self.args.host not in self.cache: - # try updating the cache - self.update_cache() - - if self.args.host not in self.cache: - # host might not exist anymore - return json_format_dict({}, True) - - return json_format_dict(self.cache[self.args.host], True) - - def _print_data(self): - data_to_print = "" - if self.args.host: - data_to_print += self.get_host_info() - else: - self.inventory['_meta'] = {'hostvars': {}} - for hostname in self.cache: - self.inventory['_meta']['hostvars'][hostname] = { - 'foreman': self.cache[hostname], - 'foreman_params': self.params[hostname], - } - if self.want_ansible_ssh_host and 'ip' in self.cache[hostname]: - self.inventory['_meta']['hostvars'][hostname]['ansible_ssh_host'] = self.cache[hostname]['ip'] - if self.want_facts: - self.inventory['_meta']['hostvars'][hostname]['foreman_facts'] = self.facts[hostname] - - data_to_print += json_format_dict(self.inventory, True) - - print(data_to_print) - - def run(self): - # Read settings and parse CLI arguments - if not self.read_settings(): - return False - self.parse_cli_args() - self.get_inventory() - self._print_data() - return True - - -if __name__ == '__main__': - sys.exit(not ForemanInventory().run()) |