Initial import of the WIP plugin

.gitignore

@@ -0,0 +1 @@
+__pycache__

ansible.cfg

@@ -0,0 +1,3 @@
+[defaults]
+inventory			= example/
+jinja2_extensions		= jinja2.ext.do

example/00-data.yml

@@ -0,0 +1,88 @@
+all:
+  hosts:
+
+    localhost:
+
+    # All of this should obviously come from some other inventory plugin.
+
+    evil-vm:
+      inv__data:
+        network: death
+        service: evil
+        instance: chaos
+
+    vm00:
+      inv__data:
+        network: dev
+        service: ldap
+        instance: dev
+        component: front
+        fostack: 1
+    vm01:
+      inv__data:
+        network: dev
+        service: ldap
+        instance: dev
+        component: front
+        fostack: 2
+    vm02:
+      inv__data:
+        network: dev
+        service: ldap
+        instance: dev
+        component: back
+        subcomponent: ro
+        fostack: 1
+    vm03:
+      inv__data:
+        network: dev
+        service: ldap
+        instance: dev
+        component: back
+        subcomponent: ro
+        fostack: 2
+    vm04:
+      inv__data:
+        network: dev
+        service: ldap
+        instance: dev
+        component: back
+        subcomponent: rw
+
+    vm05:
+      inv__data:
+        network: infra
+        service: ldap
+        instance: prod
+        component: front
+        fostack: 1
+    vm06:
+      inv__data:
+        network: infra
+        service: ldap
+        instance: prod
+        component: front
+        fostack: 2
+    vm07:
+      inv__data:
+        network: infra
+        service: ldap
+        instance: prod
+        component: back
+        subcomponent: ro
+        fostack: 1
+    vm08:
+      inv__data:
+        network: infra
+        service: ldap
+        instance: prod
+        component: back
+        subcomponent: ro
+        fostack: 2
+    vm09:
+      inv__data:
+        network: infra
+        service: ldap
+        instance: dev
+        component: back
+        subcomponent: rw

example/01-test-reconstructed.yml

@@ -0,0 +1,129 @@
+---
+plugin: reconstructed
+instructions:
+
+  # Check whether that host is managed
+  - action: set_fact
+    name: inv__managed
+    value: >-
+      {{ inv__data is defined
+         and inv__data.network is defined
+         and inv__data.service is defined
+         and inv__data.instance is defined }}
+  - when: not inv__managed
+    action: stop
+
+  # Fail when the host name starts with "evil".
+  - when: inventory_hostname.startswith( 'evil' )
+    action: fail
+    msg: "{{ inventory_hostname }} is obviously evil, skipping."
+
+  # Only create the managed groups if we *have* managed hosts
+  - loop: [managed, by_environment, by_network, by_failover_stack, by_service]
+    action: create_group
+    group: "{{ item }}"
+  - loop: [by_environment, by_network, by_failover_stack, by_service]
+    action: add_child
+    group: managed
+    child: "{{ item }}"
+
+  # Copy inv__data fields to separate inv__ variables
+  - loop:
+      - component
+      - description
+      - fostack
+      - instance
+      - network
+      - service
+      - subcomponent
+    when: inv__data[item] is defined
+    action: set_fact
+    name: "inv__{{ item }}"
+    value: "{{ inv__data[ item ] }}"
+
+  # Environment variable and groups
+  - action: set_fact
+    name: inv__environment
+    value: >-
+      {{
+        inv__data.environment | default(
+          ( inv__instance == "prod" ) | ternary( "prod", "dev" )
+        )
+      }}
+  - action: create_group
+    group: "env_{{ inv__environment }}"
+  - action: add_child
+    group: by_environment
+    child: "env_{{ inv__environment }}"
+  - action: add_host
+    group: "env_{{ inv__environment }}"
+
+  # Failover stack group
+  - action: set_var
+    name: failover_group
+    value: >-
+      {{
+        ( inv__fostack is defined )
+        | ternary( "fostack_" ~ inv__fostack | default("") , "no_failover" )
+      }}
+  - action: create_group
+    group: "{{ failover_group }}"
+  - action: add_child
+    group: by_failover_stack
+    child: "{{ failover_group }}"
+  - action: add_host
+    group: "{{ failover_group }}"
+
+  # Network group
+  - action: set_var
+    name: network_group
+    value: "net_{{ inv__network }}"
+  - action: create_group
+    group: "{{ network_group }}"
+  - action: add_child
+    group: by_network
+    child: "{{ network_group }}"
+  - action: add_host
+    group: "{{ network_group }}"
+
+  # Service group
+  - action: set_var
+    name: service_group
+    value: "svc_{{ inv__service }}"
+  - action: create_group
+    group: "{{ service_group }}"
+  - action: add_child
+    group: by_service
+    child: "{{ service_group }}"
+
+  # Component group. We add the host directly if there is no subcomponent.
+  - when: inv__component is defined
+    action: set_var
+    name: comp_group
+    value: "svcm_{{ inv__service }}_{{ inv__component }}"
+  - when: inv__component is defined
+    action: create_group
+    group: "{{ comp_group }}"
+  - when: inv__component is defined
+    action: add_child
+    group: "{{ service_group }}"
+    child: "{{ comp_group }}"
+  - when: inv__component is defined and inv__subcomponent is not defined
+    action: add_host
+    group: "{{ comp_group }}"
+
+  # Subcomponent group.
+  - when: inv__component is defined and inv__subcomponent is defined
+    action: set_var
+    name: subcomp_group
+    value: "svcm_{{ inv__service }}_{{ inv__subcomponent }}"
+  - when: inv__component is defined and inv__subcomponent is defined
+    action: create_group
+    group: "{{ subcomp_group }}"
+  - when: inv__component is defined and inv__subcomponent is defined
+    action: add_child
+    group: "{{ comp_group }}"
+    child: "{{ subcomp_group }}"
+  - when: inv__component is defined and inv__subcomponent is defined
+    action: add_host
+    group: "{{ subcomp_group }}"

inventory_plugins/reconstructed.py

@@ -0,0 +1,422 @@
+from ansible import constants as C
+from ansible.errors import AnsibleParserError, AnsibleRuntimeError, AnsibleError
+from ansible.module_utils.six import string_types
+from ansible.module_utils.parsing.convert_bool import boolean
+from ansible.utils.vars import isidentifier
+from ansible.plugins.inventory import BaseInventoryPlugin
+
+DOCUMENTATION = """
+    name: reconstructed
+    short_description: A plugin that allows the dynamic construction of groups
+    author: Emmanuel BENOÎT
+    description:
+    - This inventory plugin allows the construction of groups, the optional
+      assignment of hosts to these groups and the computation of arbitrary
+      facts.
+    options:
+      plugin:
+        description:
+        - Token that ensures this is a source file for the C(group_creator)
+          plugin.
+        required: True
+        choices: ['reconstructed']
+      instructions:
+        description:
+        - The list of instructions to be executed in order to generate the
+          inventory parts. Each instruction is represented as a dictionnary
+          with at least an C(action) field which determines which instruction
+          must be executed. The instructions will be executed once for each
+          inventory host.
+        - Instructions may include various fields that act as control flow.
+        - If the C(loop) field is present, it must contain a list (or a Jinja
+          template that will return a list). The instruction will be repeated
+          for each value in the list. The C(loop_var) field may be added to
+          specify the name of the variable into which the current value will
+          be written; by default the C(item) variable will be used.
+        - The C(when) field, if present, must contain a Jinja expression
+          representing a condition which will be checked before the instruction
+          is executed.
+        - The C(action) field must be set to one of the following values.
+        - C(create_group) creates a group. The name of the group must be
+          provided using the C(group) field, which must be a valid name or a
+          Jinja template that evaluates to a valid name.
+        - C(add_child) adds a child group to another group. The name of the
+          group being added must be provided in the C(child) entry, while
+          the name of the parent must be provided in the C(group) entry. Both
+          groups must exist. In addition, the names may be specified using
+          Jinja templates.
+        - C(add_host) adds the current inventory host to a group. The name
+          of the group must be provided in the C(group) entry. The group
+          must exist.
+        - C(fail) causes the computations for the current host to stop with
+          an error. The error message may be specified in the C(message)
+          entry; if present, it will be evaluated using Jinja.
+        - C(set_fact) and C(set_var) create a fact and a local variable,
+          respectively. Local variables will only be kept during the execution
+          of the script for the current host, while facts will be added to the
+          host's data. The C(name) entry specifies the name of the fact or
+          variable while the C(value) entry specifies its value. Both may be
+          Jinja templates.
+        - C(stop) stops processing the list of instructions for the current
+          host.
+        type: list
+        elements: dict
+        required: True
+      strictness:
+        description:
+        - The C(host) setting will cause an error to skip the host being
+          processed, and the C(full) setting will abort the execution
+          altogether.
+        required: False
+        choices: ['host', 'full']
+        default: host
+"""
+
+
+class RcInstruction:
+    """An instruction that can be executed by the plugin."""
+
+    COMMON_FIELDS = ("when", "loop", "loop_var", "action")
+    DEFAULT_LOOP_VAR = "item"
+
+    def __init__(self, inventory, templar, action, allowed_fields=()):
+        self._inventory = inventory
+        self._templar = templar
+        self._condition = None
+        self._loop = None
+        self._loop_var = None
+        self._action = action
+        self._allowed_fields = set(allowed_fields)
+        self._allowed_fields.update(RcInstruction.COMMON_FIELDS)
+
+    def parse(self, record):
+        assert "action" in record and record["action"] == self._action
+        # Ensure there are no unsupported fields
+        extra_fields = set(record.keys()).difference(self._allowed_fields)
+        if extra_fields:
+            raise AnsibleParserError(
+                "%s: unsupported fields: %s" % (self._action, ", ".join(extra_fields))
+            )
+        # Extract the condition
+        if "when" in record:
+            if not isinstance(record["when"], string_types):
+                raise AnsibleParserError(
+                    "%s: 'when' clause is not a string" % (self._action,)
+                )
+            self._condition = record["when"]
+        # Extract the loop data and configuration
+        if "loop" in record:
+            loop = record["loop"]
+            if not isinstance(loop, string_types + (list,)):
+                raise AnsibleParserError(
+                    "%s: 'loop' clause is neither a string nor a list" % (self._action,)
+                )
+            loop_var = record.get("loop_var", RcInstruction.DEFAULT_LOOP_VAR)
+            if not isinstance(loop_var, string_types):
+                raise AnsibleParserError(
+                    "%s: 'loop_var' clause is not a string" % (self._action,)
+                )
+            if not isidentifier(loop_var):
+                raise AnsibleParserError(
+                    "%s: 'loop_var' value '%s' is not a valid identifier"
+                    % (self._action, loop_var)
+                )
+            self._loop = loop
+            self._loop_var = loop_var
+        elif "loop_var" in record:
+            raise AnsibleParserError(
+                "%s: 'loop_var' clause found without 'loop'" % (self._action,)
+            )
+        # Process action-specific fields
+        self.parse_action(record)
+
+    def parse_group_name(self, record, name):
+        if name not in record:
+            raise AnsibleParserError("%s: missing '%s' field" % (self._action, name))
+        group = record[name]
+        if not isinstance(group, string_types):
+            raise AnsibleParserError(
+                "%s: '%s' field must be a string" % (self._action, name)
+            )
+        may_be_template = self._templar.is_possibly_template(group)
+        if not may_be_template:
+            group = group.strip()
+            if C.INVALID_VARIABLE_NAMES.findall(group):
+                raise AnsibleParserError(
+                    "%s: invalid group name '%s' in field '%s'"
+                    % (self._action, group, name)
+                )
+        return may_be_template, group
+
+    def parse_action(self, record):
+        raise NotImplementedError
+
+    def run_for(self, host_name, host_vars, script_vars):
+        merged_vars = host_vars.copy()
+        merged_vars.update(script_vars)
+        if self._loop is None:
+            return self.run_once(host_name, merged_vars, host_vars, script_vars)
+        loop_values = self.evaluate_loop(host_name, merged_vars)
+        for value in loop_values:
+            merged_vars[self._loop_var] = value
+            if not self.run_once(host_name, merged_vars, host_vars, script_vars):
+                return False
+        return True
+
+    def run_once(self, host_name, merged_vars, host_vars, script_vars):
+        if self.evaluate_condition(host_name, merged_vars):
+            return self.execute_action(host_name, merged_vars, host_vars, script_vars)
+        else:
+            return True
+
+    def evaluate_condition(self, host_name, variables):
+        if self._condition is None:
+            return True
+        t = self._templar
+        t.available_variables = variables
+        template = "%s%s%s" % (
+            t.environment.variable_start_string,
+            self._condition,
+            t.environment.variable_end_string,
+        )
+        return boolean(t.template(template, disable_lookups=False))
+
+    def evaluate_loop(self, host_name, variables):
+        if isinstance(self._loop, list):
+            return self._loop
+        assert isinstance(self._loop, string_types)
+        self._templar.available_variables = variables
+        value = self._templar.template(self._loop, disable_lookups=False)
+        if not isinstance(value, list):
+            raise AnsibleRuntimeError(
+                "template '%s' did not evaluate to a list" % (self._loop,)
+            )
+        return value
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        raise NotImplementedError
+
+    def get_templated_group(self, variables, may_be_template, name, must_exist=False):
+        if may_be_template:
+            self._templar.available_variables = variables
+            real_name = self._templar.template(name)
+            if not isinstance(name, string_types):
+                raise AnsibleRuntimeError(
+                    "%s: '%s' did not coalesce into a string" % (self._action, name)
+                )
+            real_name = real_name.strip()
+            if C.INVALID_VARIABLE_NAMES.findall(real_name):
+                raise AnsibleRuntimeError(
+                    "%s: '%s' is not a valid group name" % (self._action, real_name)
+                )
+        else:
+            real_name = name
+        if must_exist and real_name not in self._inventory.groups:
+            raise AnsibleRuntimeError(
+                "%s: group '%s' does not exist" % (self._action, real_name)
+            )
+        return real_name
+
+
+class RciCreateGroup(RcInstruction):
+    def __init__(self, inventory, templar):
+        super().__init__(inventory, templar, "create_group", ("group",))
+        self._may_be_template = None
+        self._group = None
+
+    def parse_action(self, record):
+        assert self._may_be_template is None and self._group is None
+        self._may_be_template, self._group = self.parse_group_name(record, "group")
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        assert not (self._may_be_template is None or self._group is None)
+        name = self.get_templated_group(merged_vars, self._may_be_template, self._group)
+        self._inventory.add_group(name)
+        return True
+
+
+class RciAddHost(RcInstruction):
+    def __init__(self, inventory, templar):
+        super().__init__(inventory, templar, "add_host", ("group",))
+        self._may_be_template = None
+        self._group = None
+
+    def parse_action(self, record):
+        assert self._may_be_template is None and self._group is None
+        self._may_be_template, self._group = self.parse_group_name(record, "group")
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        assert not (self._may_be_template is None or self._group is None)
+        name = self.get_templated_group(
+            merged_vars, self._may_be_template, self._group, must_exist=True
+        )
+        self._inventory.add_child(name, host_name)
+        return True
+
+
+class RciAddChild(RcInstruction):
+    def __init__(self, inventory, templar):
+        super().__init__(inventory, templar, "add_child", ("group", "child"))
+        self._group_mbt = None
+        self._group_name = None
+        self._child_mbt = None
+        self._child_name = None
+
+    def parse_action(self, record):
+        assert self._group_mbt is None and self._group_name is None
+        assert self._child_mbt is None and self._child_name is None
+        self._group_mbt, self._group_name = self.parse_group_name(record, "group")
+        self._child_mbt, self._child_name = self.parse_group_name(record, "child")
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        assert not (self._group_mbt is None or self._group_name is None)
+        assert not (self._child_mbt is None or self._child_name is None)
+        group = self.get_templated_group(
+            merged_vars, self._group_mbt, self._group_name, must_exist=True
+        )
+        child = self.get_templated_group(
+            merged_vars, self._child_mbt, self._child_name, must_exist=True
+        )
+        self._inventory.add_child(group, child)
+        return True
+
+
+class RciSetVarOrFact(RcInstruction):
+    def __init__(self, inventory, templar, is_fact):
+        action = "set_" + ("fact" if is_fact else "var")
+        super().__init__(inventory, templar, action, ("name", "value"))
+        self._is_fact = is_fact
+        self._var_name = None
+        self._name_may_be_template = None
+        self._var_value = None
+
+    def parse_action(self, record):
+        assert (
+            self._var_name is None
+            and self._name_may_be_template is None
+            and self._var_value is None
+        )
+        if "name" not in record:
+            raise AnsibleParserError("%s: missing 'name' field" % (self._action,))
+        name = record["name"]
+        if not isinstance(name, string_types):
+            raise AnsibleParserError("%s: 'name' must be a string" % (self._action,))
+        if "value" not in record:
+            raise AnsibleParserError("%s: missing 'value' field" % (self._action,))
+        nmbt = self._templar.is_possibly_template(name)
+        if not (nmbt or isidentifier(name)):
+            raise AnsibleParserError(
+                "%s: '%s' is not a valid variable name" % (self._action, name)
+            )
+        self._name_may_be_template = nmbt
+        self._var_name = name
+        self._var_value = record["value"]
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        assert not (
+            self._var_name is None
+            or self._name_may_be_template is None
+            or self._var_value is None
+        )
+        self._templar.available_variables = merged_vars
+        if self._name_may_be_template:
+            name = self._templar.template(self._var_name)
+            if not isinstance(name, string_types):
+                raise AnsibleRuntimeError(
+                    "%s: '%s' did not coalesce into a string"
+                    % (self._action, self._var_name)
+                )
+            if not isidentifier(name):
+                raise AnsibleRuntimeError(
+                    "%s: '%s' is not a valid variable name" % (self._action, name)
+                )
+        else:
+            name = self._var_name
+        value = self._templar.template(self._var_value)
+        merged_vars[name] = value
+        if self._is_fact:
+            self._inventory.set_variable(host_name, name, value)
+            host_vars[name] = value
+        else:
+            script_vars[name] = value
+        return True
+
+
+class RciStop(RcInstruction):
+    def __init__(self, inventory, templar):
+        super().__init__(inventory, templar, "stop")
+
+    def parse_action(self, record):
+        pass
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        return False
+
+
+class RciFail(RcInstruction):
+    def __init__(self, inventory, templar):
+        super().__init__(inventory, templar, "fail", ("msg",))
+        self._message = None
+
+    def parse_action(self, record):
+        self._message = record.get("msg", None)
+
+    def execute_action(self, host_name, merged_vars, host_vars, script_vars):
+        if self._message is None:
+            message = "fail requested (%s)" % (host_name,)
+        else:
+            self._templar.available_variables = merged_vars
+            message = self._templar.template(self._message)
+        raise AnsibleRuntimeError(message)
+
+
+INSTRUCTIONS = {
+    "add_child": RciAddChild,
+    "add_host": RciAddHost,
+    "create_group": RciCreateGroup,
+    "fail": RciFail,
+    "set_fact": lambda i, t: RciSetVarOrFact(i, t, True),
+    "set_var": lambda i, t: RciSetVarOrFact(i, t, False),
+    "stop": RciStop,
+}
+
+
+class InventoryModule(BaseInventoryPlugin):
+    """Constructs groups based on lists of instructions."""
+
+    NAME = "reconstructed"
+
+    def verify_file(self, path):
+        return super().verify_file(path) and path.endswith((".yaml", ".yml"))
+
+    def parse(self, inventory, loader, path, cache=True):
+        super().parse(inventory, loader, path, cache)
+        self._read_config_data(path)
+        instr_src = self.get_option("instructions")
+        instructions = []
+        for record in instr_src:
+            instructions.append(self.get_instruction(record))
+        for host in inventory.hosts:
+            try:
+                self.exec_for_host(host, instructions)
+            except AnsibleError as e:
+                if self.get_option("strictness") == "full":
+                    raise
+                self.display.warning(
+                    "reconstructed - error on host %s: %s" % (host, repr(e))
+                )
+
+    def exec_for_host(self, host, instructions):
+        host_vars = self.inventory.get_host(host).get_vars()
+        script_vars = {}
+        for instruction in instructions:
+            if not instruction.run_for(host, host_vars, script_vars):
+                return
+
+    def get_instruction(self, record):
+        action = record["action"]
+        if action not in INSTRUCTIONS:
+            raise AnsibleParserError("Unknown action '%s'" % (action,))
+        instruction = INSTRUCTIONS[action](self.inventory, self.templar)
+        instruction.parse(record)
+        return instruction