add src and scripts for packe

scripts/packme

+#!/usr/bin/env python3
+
+"""
+packme is a python3 application for generating and running packer templates.
+
+It works by generating packer manifest.json files out of manifest.yml files which contain the packer settings for the image to build.
+"""
+
+import argparse
+import glob
+import os
+import shutil
+import sys
+
+from packman import Packman
+
+def clean_templates_dir(templates_dir):
+    """Removes all manifest.json files, packer_cache and builds directories found in
+    templates directory
+    """
+
+    for template_dir in templates_dir:
+
+        try:
+            os.remove(os.path.join(template_dir,"manifest.json"))
+        except FileNotFoundError:
+            pass
+
+        try:
+            shutil.rmtree(os.path.join(template_dir,"builds"))
+        except FileNotFoundError:
+            pass
+
+        try:
+            shutil.rmtree(os.path.join(template_dir,"packer_cache"))
+        except FileNotFoundError:
+            pass
+
+def parse_args():
+    """Setup and run the argument parser.
+    """
+
+    parser = argparse.ArgumentParser(description="This is packme: a program for generating and running packer configuration(s)")
+    parser.add_argument("--log", "-l", dest="log", action="store_true", help="log packer output")
+    parser.add_argument("--clean", "-c", dest="clean", action="store_true", help="clean up templates directories")
+    parser.add_argument("--debug", "-d", dest="debug", action="store_true", help="debug mode (keep temporary files)")
+    parser.add_argument("--run", "-r", dest="run", action="store_true", help="run packer after json manifest(s) generation")
+    parser.add_argument("--templates-dir", "-t", dest="templates_dir", required=True, help="Templates base directory")
+    parser.add_argument("--selected-templates", "-s", dest="selected_templates", nargs="*", default=["*"], help="run packman on selected template(s)")
+    parser.add_argument("--input", "-i", dest="input_file", help="YAML input file")
+
+    args = parser.parse_args()
+
+    return args
+                
+if __name__ == "__main__":
+
+    # Parse command line arguments and fetch their values
+    args = parse_args()
+    input_file = args.input_file
+    run = args.run
+    templates_dir = args.templates_dir
+    selected_templates = args.selected_templates
+    clean = args.clean
+    debug = args.debug
+    log = args.log
+
+    # If --clean option is set, cleanup the templates dir (builds, packer_cache, manifest.json files and directories)
+    if clean:
+        clean_templates_dir(templates_dir)
+
+    if not input_file:
+        sys.exit(0)
+
+    t = Packman.Packman(input_file, templates_dir)
+
+    t.build(selected_templates=selected_templates, indent=4, separators=(',', ': '))
+
+    if run:
+        t.run(selected_templates=selected_templates, log=log)
+
+    # If --debug option is set, keep temporary files and directories in templates dir (builds, packer_cache, manifest.json)
+    if not debug:
+        clean_templates_dir(templates_dir)
+
+

src/packman.egg-info/PKG-INFO

+Metadata-Version: 2.1
+Name: packman
+Version: 0.0.0
+Summary: Heler app for building and running packer templates
+Home-page: https://code.ill.fr/si/packer
+Author: Eric Pellegrini
+Author-email: pellegrini@ill.fr
+Maintainer: Eric Pellegrini
+Maintainer-email: pellegrini@ill.fr
+License: MIT
+Description: Overview
+        ========
+        **packman** is a python3 application for generating and running packer templates out of YAML configuration file.
+        
+        Prerequesites
+        =============
+        - python3 + pip
+        - packer application must be installed
+        
+        Installation
+        ============
+        see [here](https://code.ill.fr/si/packer) for complementary info
+        
+        - `cd` to the directory where lies the `setup.py` file
+        - pip3 install --user .
+        
+        
+Platform: Unix
+Platform: Windows
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown

src/packman.egg-info/SOURCES.txt

+README.rst
+setup.py
+scripts/packme
+src/packman/PackerTemplate.py
+src/packman/Packman.py
+src/packman/__init__.py
+src/packman/__pkginfo__.py
+src/packman.egg-info/PKG-INFO
+src/packman.egg-info/SOURCES.txt
+src/packman.egg-info/dependency_links.txt
+src/packman.egg-info/requires.txt
+src/packman.egg-info/top_level.txt
\ No newline at end of file

src/packman.egg-info/dependency_links.txt

+

src/packman.egg-info/requires.txt

+pyyaml
+Jinja2

src/packman.egg-info/top_level.txt

+packman

src/packman/PackerTemplate.py

+#!/usr/bin/env python3
+
+#################################################################
+# This apps allows the creation of packer json file out of
+# YAML template
+# One of its main feature is to handle template inheritance
+# which is not covered yet natively by packer
+# To run the application:
+# env_var_1 env_var_2 ... env_var_n rackman.py template.yml
+# where env_var_1 ... env_var_n are environment variables
+# passed at the command line level and template.yml is the YAML
+# which contain the packer configuration
+#################################################################
+
+import collections
+import glob
+import jinja2
+import json
+import os
+import pprint
+import yaml
+
+class PackerTemplate:
+    """This class implements one packer template.
+    Basically, a packer template is made of four sections:
+    - description: a string that explains the purpose of the template
+    - variables: a dictionary of variables used in the template through jinja2 mechanism
+    - builders: a list of dictionaries where each dictionary defines the type of image that will be built
+    - provisioners: a list of dictionaries where each dictionary defines actions used to configure the image
+    - processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
+    """
+
+    def __init__(self, name, yaml_node, packages):
+
+        self._name = name
+
+        self._parameters = yaml_node.get("parameters", {})
+
+        # Fetch the 'packer' node
+        packer_node = yaml_node.get("packer", {})
+
+        # Fetch the 'description' node from the packer node
+        self._description = packer_node.get("description", "No description provided")
+
+        # Fetch the '_variables' node from the packer node
+        self._variables = packer_node.get("variables", {})
+
+        # Fetch the 'builders' node from the packer node
+        self._builders = packer_node.get("builders", [])
+
+        # Fetch the 'provisioners' node from the packer node
+        self._provisioners = packer_node.get("provisioners", [])
+
+        # Fetch the 'postprocessors' node from the packer node
+        self._postprocessors = packer_node.get("postprocessors", [])
+
+        bin_dir = os.path.realpath(os.path.dirname(__file__))
+        root_dir = os.path.dirname(bin_dir)
+        self._packages_root_dir = os.path.join(root_dir,"packages")
+        self._templates_dir = os.path.join(root_dir,"templates")
+
+        self._load_packages(packages)
+
+    @property
+    def builders(self):
+        """Property for _builders attribute.
+
+        Returns
+        -------
+        list of packer builders
+            The list of packer builders for this PackerTemplate.
+        """
+
+        return self._builders
+
+    @property
+    def description(self):
+        """Property for _description attribute.
+
+        Returns
+        -------
+        str
+            The description of this PackerTemplate.
+        """
+
+        return self._description
+
+    @property
+    def name(self):
+        """Property for _name attribute.
+
+        Returns
+        -------
+        str
+            The name of this PackerTemplate.
+        """
+
+        return self._name
+
+    @property
+    def parameters(self):
+        """Property for _parameters attribute.
+
+        Returns
+        -------
+        dict
+            The parameters that will be used by Jinja 2 when creating the template json file.
+        """
+
+        return self._parameters
+
+    @property
+    def postprocessors(self):
+        """Property for _postprocessors attribute.
+
+        Returns
+        -------
+        list of packer postprocessors
+            The list of packer postprocessors for this PackerTemplate.
+        """
+
+        return self._postprocessors
+
+    @property
+    def provisioners(self):
+        """Property for _provisioners attribute.
+
+        Returns
+        -------
+        list of packer provisioners
+            The list of packer provisioners for this PackerTemplate.
+        """
+
+        return self._provisioners
+
+    @property
+    def variables(self):
+        """Property for _variables attribute.
+
+        Returns
+        -------
+        list of packer variables
+            The list of packer variables for this PackerTemplate.
+        """
+
+        return self._variables
+
+    def set_parent(self, parent_template):
+        """Set the parent template to this PackerTemplate.
+
+        This defines a relationship for future packer run in the sense that the child template will start directly from the image of 
+        its parent template.
+
+        Parameters
+        ----------
+        parent_template: :obj: `PackerTemplate`
+            The PackerTemplate of the parent template to connect the child template with.
+            
+        """
+
+        # List of the builder names for the child template
+        child_builder_names = [builder["name"] for builder in self._builders]
+
+        # Loop over the builder of the parent template
+        for builder in parent_template.builders:
+
+            builder_name = builder["name"]
+
+            # If this is a builder specific to the parent config, copy it in the child config and set it with an image dependency
+            if builder_name not in child_builder_names:
+                parent_builder = {}
+                parent_builder["name"] = builder_name
+                parent_builder["type"] = builder["type"]
+                parent_builder["vm_name"] = self._name
+                parent_builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
+                parent_builder["iso_checksum_type"] = "none"
+                parent_builder["iso_checksum_url"] = "none"
+                parent_builder["output_directory"] = os.path.join(self._templates_dir,self._name)
+                self._builders.insert(0,parent_builder)
+
+            # If the builder is also defined in the child config, use the child config one and specify the image dependency
+            else:
+
+                builder = next((b for b in self._builders if b["name"] == builder_name),None)
+                if builder is None:
+                    continue
+
+                builder["vm_name"] = self._name
+                builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
+                builder["iso_checksum_type"] = "none"
+                builder["iso_checksum_url"] = "none"
+                builder["output_directory"] = os.path.join(self._templates_dir,self._name)            
+
+    def _load_packages(self, packages):
+        """Load the non-standard package YAML file and append them as provisioners of this PackerTemplate.
+
+        Parameters
+        ----------
+        list of str
+            The list of the packages to append.    
+        """
+
+        # If *" is in the list, fetch all the packages
+        if "*" in packages:
+            packages_dir = glob.glob(os.path.join(self._packages_root_dir,"*"))
+        # Otherwise just fetch the selected ones
+        else:
+            packages_dir = []
+            for package in packages:
+                package_dir = os.path.join(self._packages_dir,package)
+                if os.path.exists(package_dir) and os.path.isdir(package_dir):
+                    packages_dir.append(package_dir)
+
+        # Loop over the packages directories
+        for package_dir in packages_dir:
+            
+            # Build the path to the package manifest file (YAML)
+            manifest_file = os.path.join(package_dir,"manifest.yml")
+            
+            # Open and load the provisioners list from the manifest file
+            try:
+              fin = open(manifest_file, "r")
+            # If the manifest does not exist, the provisioners list is set to an empty list
+            except FileNotFoundError:
+                manifest_data = []
+            else:
+                manifest_data = yaml.safe_load(fin)
+                manifest_data = manifest_data["provisioners"]
+
+            # Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
+            for provisioner in manifest_data:
+                if provisioner["type"] in ["file"]:
+                    provisioner["source"] = os.path.join(package_dir,provisioner["source"])
+                elif provisioner["type"] in ["shell"]:
+                    provisioner["script"] = os.path.join(package_dir,provisioner["script"])
+
+            # Extend the current provisioners list with the ones of the selected packages
+            self._provisioners.extend(manifest_data)
+            
+    def dump(self, output_file, **kwargs):
+        """Dump this PackerTemplate to a file.
+
+        Parameters
+        ----------
+        output_file: str
+            The path to the output json file for this PackerTemplate.
+        """
+
+        # Get the basename and the ext of the output_file
+        basename, ext = os.path.splitext(output_file)
+        # If the ext is different from .json set it to .json
+        if ext != ".json":
+            ext = ".json"
+
+        # Reformat the output_file
+        output_file = "{}{}".format(basename, ext)
+
+        # This will be the node to be dumped
+        node = {}
+        node["description"] = self._description
+        node["variables"] = self._variables
+        node["builders"] = self._builders
+        node["provisioners"] = self._provisioners
+        node["post-processors"] = self._postprocessors
+
+        # Render the jinja2 templates with the parameters dictionary provided in the template file and the available environment variables
+        jinja_template = jinja2.Template(repr(node))
+        s = jinja_template.render(parameters=self._parameters, environment=os.environ)
+
+        # Dump to the output file
+        with open(output_file, "w") as fout:
+            json.dump(yaml.safe_load(s), fout, **kwargs)
+
+    def __str__(self):
+        """Returns fancy output for this PackerTemplate.
+
+        Returns
+        -------
+        str
+            The string that will result from a str call to a PackerTemplate object.
+        """
+
+        d = collections.OrderedDict()
+        d["description"] = self._description
+        d["variables"] = self._variables
+        d["builders"] = self._builders
+        d["provisioners"] = self._provisioners
+        d["post-processors"] = self._postprocessors
+
+        return pprint.pformat(d)
+
+
+

src/packman/Packman.py

+#!/usr/bin/env python3
+
+#################################################################
+# This apps allows the creation of packer json file out of
+# YAML template
+# One of its main feature is to handle template inheritance
+# which is not covered yet natively by packer
+# To run the application:
+# env_var_1 env_var_2 ... env_var_n rackman.py template.yml
+# where env_var_1 ... env_var_n are environment variables
+# passed at the command line level and template.yml is the YAML
+# which contain the packer configuration
+#################################################################
+
+import collections
+import os
+import subprocess
+import yaml
+
+from .PackerTemplate import PackerTemplate
+
+class Packman:
+    """This class implements the Packman engine for generating packer template json files and run packer optionally .
+    """
+
+    def __init__(self, input_file, templates_dir):
+        """Constructor.
+
+        Parameters
+        ----------
+        input_file: str
+            Path to the Packman input file.
+        """
+
+        # The base directory for templates
+        self._templates_dir = os.path.abspath(templates_dir)
+
+        with open(input_file, "r") as fin:
+            data = yaml.safe_load(fin)
+
+        # The input file must ne a YAML file which declares a 'templates' dictionary
+        if "templates" not in data:
+            raise IOError("Invalid YAML file: must contains 'templates' tag")
+
+        self._templates = data["templates"]
+
+    def get_template(self, template_name):
+        """Return the YAML contents of a given template.
+
+        Parameters
+        ----------
+        template_name: str
+            The name of the template to fetch.
+
+        Returns
+        -------
+        :obj: `PackerTemplate`
+
+            The YAML contents of the template to be fetched.
+        """
+
+        return self._templates[template_name] if isinstance(self._templates[template_name],dict) else {}
+
+    def _build_template(self, template_name):
+        """Build a PackerTemplate object from a template name.
+
+        Parameters
+        ----------
+        template_name: str
+            The name of the template to build.
+
+        Returns
+        -------
+        :obj: `PackerTemplate`
+
+            The template object used by packman to build the manifest.json file.        
+        """
+
+        # Fetch the template matching template_name key
+        template_node = self.get_template(template_name)
+
+        # Build the path for the template manifest file (YAML)
+        manifest_file = os.path.join(self._templates_dir,template_name,"manifest.yml")
+        
+        # Opens the file and load its contents
+        try:
+          fin = open(manifest_file, "r")
+        # If the file does not exist, the manifest contents is just an empty dict
+        except FileNotFoundError:
+            manifest_data = {}
+        else:
+            manifest_data = yaml.safe_load(fin)
+
+        # Get the packages node which gives the list of non-standard applications to add to the packer process 
+        packages = template_node.get("packages", [])
+
+        # Build a PackerTemplate object that can be dumped to a manifest.json file
+        template = PackerTemplate(template_name, manifest_data, packages)
+
+        return template
+
+    def _build_template_hierarchy(self, template_name, hierarchy):
+        """Build a single template hierarchy.
+
+        A template can have a parent template. In that case for packer neig able to run on those templates, 
+        the parent tenplate must have been built before.
+
+        Getting a hierarchy of templates, the first one being  the ones with no parent is the goal of this method.
+
+        Parameters
+        ----------
+        template_name: str
+            The template on which the hierarchy will be built upon.
+
+        hierarchy: list str
+            The template hierarchy.
+
+            This argument is just used for passing the template hierarchy across recursive calls of this method. 
+        """
+
+        # Fetch the template matching template_name key
+        template_node = self.get_template(template_name)
+
+        extends = template_node.get("extends",None)
+
+        hierarchy.append(template_name)
+
+        if extends is None:
+            return
+
+        else:
+            self._build_template_hierarchy(extends, hierarchy)
+            
+    def _build_config_hierarchy(self, selected_templates=None):
+        """Build the templates hierarchy.
+
+        A template can have a parent template. In that case for packer neig able to run on those templates, 
+        the parent tenplate must have been built before.
+
+        Getting a hierarchy of templates, the first one being  the ones with no parent is the goal of this method.
+
+        Parameters
+        ----------
+        selected_templates: list of str
+            List of packer templates.
+
+        Returns
+        -------
+        List of str
+            Returns the hierarchy of templates from the one with no parent to the ones with parents.
+        """
+
+        if selected_templates is None:
+            templates = set(self._templates.keys())
+        else:
+            # Filter out the image names not present in the yaml file
+            templates = set([template for template in selected_templates if template in self._templates])
+
+        config_hierarchy = []
+        for template in templates:
+            self._build_template_hierarchy(template, config_hierarchy)
+
+        config_hierarchy.reverse()
+        config_hierarchy = list(collections.OrderedDict.fromkeys(config_hierarchy))
+
+        return config_hierarchy
+
+    def run(self, selected_templates=None, log=False):
+        """Run packer on the generated manifest.json files.
+
+        Parameters
+        ----------
+        selected_templates: list of str, optional
+            List of packer templates.
+        """
+
+        # Set env variables for packer run environment
+        packer_env = os.environ.copy()
+        # This allow to speed up the typing of the preseed command line
+        packer_env["PACKER_KEY_INTERVAL"] = "10ms"
+        # This will add log output for packer run    
+        packer_env["PACKER_LOG"] = "1" if log else "0"
+
+        # Define the template hierarchy for the selected templates
+        config_hierarchy = self._build_config_hierarchy(selected_templates)
+