Packman.py 8.97 KB
Newer Older
1 2
import collections
import os
eric pellegrini's avatar
eric pellegrini committed
3
import shutil
4 5 6
import subprocess
import yaml

7
from typing import Any, Dict, List, Optional
8

9 10 11 12 13 14
from .PackerTemplate import PackerTemplate

class Packman:
    """This class implements the Packman engine for generating packer template json files and run packer optionally .
    """

15
    def __init__(self, input_file: str, templates_base_dir: str) -> None:
16 17 18 19 20 21 22 23 24
        """Constructor.

        Parameters
        ----------
        input_file: str
            Path to the Packman input file.
        """

        # The base directory for templates
25
        self._templates_base_dir = os.path.abspath(templates_base_dir)
26 27 28 29 30 31 32 33 34 35

        with open(input_file, "r") as fin:
            data = yaml.safe_load(fin)

        # The input file must ne a YAML file which declares a 'templates' dictionary
        if "templates" not in data:
            raise IOError("Invalid YAML file: must contains 'templates' tag")

        self._templates = data["templates"]

36 37
    def get_templates_selection(self, selected_templates = None):
        """Filter out a template selection from those which are not actual ones.
38 39 40 41 42 43 44 45

        Parameters
        ----------
        template_name: str
            The name of the template to fetch.

        Returns
        -------
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
        list
            The filtered templates list.
        """

        if selected_templates is None:
            selected_templates = ["*"]

        # If '*' is in the list of the selected templates, pick all the templates found in the templates base directory
        if "*" in selected_templates:
            templates = self._templates.keys()
        else:
            # Filter out the image names not present in the yaml file
            templates = [template for template in selected_templates if template in self._templates]

        return templates

    def get_template(self, template_name):
        """Return the YAML contents of a given template.
64
        :class:`.PackerTemplate`
65

66 67 68 69 70 71 72 73 74
        Parameters
        ----------
        selected_templates: list, optional
            List of strings corresponding to the packer templates from which the hierarchy should be built.

        Returns
        -------
        list
            Returns the hierarchy of templates from the one with no parent to the ones with parents.
75 76 77 78
        """

        return self._templates[template_name] if isinstance(self._templates[template_name],dict) else {}

79
    def _build_template(self, template_name : str) -> PackerTemplate:
80 81 82 83 84 85 86 87 88
        """Build a PackerTemplate object from a template name.

        Parameters
        ----------
        template_name: str
            The name of the template to build.

        Returns
        -------
89
        :class:`.PackerTemplate`
90 91 92 93 94 95 96 97

            The template object used by packman to build the manifest.json file.        
        """

        # Fetch the template matching template_name key
        template_node = self.get_template(template_name)

        # Build the path for the template manifest file (YAML)
98
        manifest_file = os.path.join(self._templates_base_dir,template_name,"manifest.yml")
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
        
        # Opens the file and load its contents
        try:
          fin = open(manifest_file, "r")
        # If the file does not exist, the manifest contents is just an empty dict
        except FileNotFoundError:
            manifest_data = {}
        else:
            manifest_data = yaml.safe_load(fin)

        # Get the packages node which gives the list of non-standard applications to add to the packer process 
        packages = template_node.get("packages", [])

        # Build a PackerTemplate object that can be dumped to a manifest.json file
        template = PackerTemplate(template_name, manifest_data, packages)

        return template

117
    def _build_template_hierarchy(self, template_name : str, hierarchy : List[str]):
118 119 120 121 122 123 124 125 126 127 128 129
        """Build a single template hierarchy.

        A template can have a parent template. In that case for packer neig able to run on those templates, 
        the parent tenplate must have been built before.

        Getting a hierarchy of templates, the first one being  the ones with no parent is the goal of this method.

        Parameters
        ----------
        template_name: str
            The template on which the hierarchy will be built upon.

eric pellegrini's avatar
eric pellegrini committed
130 131
        hierarchy: list
            A list of strings corresponding tot the template hierarchy.
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148

            This argument is just used for passing the template hierarchy across recursive calls of this method. 
        """

        # Fetch the template matching template_name key
        template_node = self.get_template(template_name)

        extends = template_node.get("extends",None)

        hierarchy.append(template_name)

        if extends is None:
            return

        else:
            self._build_template_hierarchy(extends, hierarchy)
            
149
    def _build_config_hierarchy(self, selected_templates : Optional[List[str]] = None):
150 151 152 153 154 155 156 157 158
        """Build the templates hierarchy.

        A template can have a parent template. In that case for packer neig able to run on those templates, 
        the parent tenplate must have been built before.

        Getting a hierarchy of templates, the first one being  the ones with no parent is the goal of this method.

        Parameters
        ----------
eric pellegrini's avatar
eric pellegrini committed
159 160
        selected_templates: list, optional
            List of strings corresponding to the packer templates from which the hierarchy should be built.
161 162 163

        Returns
        -------
164
        list
165 166 167
            Returns the hierarchy of templates from the one with no parent to the ones with parents.
        """

168
        templates = self.get_templates_selection(selected_templates)
169

170
        config_hierarchy : List[str] = []
171 172 173 174 175 176 177 178
        for template in templates:
            self._build_template_hierarchy(template, config_hierarchy)

        config_hierarchy.reverse()
        config_hierarchy = list(collections.OrderedDict.fromkeys(config_hierarchy))

        return config_hierarchy

179
    def run(self, selected_templates : Optional[List[str]] = None, log : Optional[bool] = False):
180 181 182 183
        """Run packer on the generated manifest.json files.

        Parameters
        ----------
eric pellegrini's avatar
eric pellegrini committed
184
        selected_templates: list, optional
185
            The packer templates to run with packer.
186 187
        """

eric pellegrini's avatar
eric pellegrini committed
188 189 190 191
        # Check first that packer program is installed somewhere
        if shutil.which("packer") is None:
            raise FileNotFoundError("packer could not be found.")

192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
        # Set env variables for packer run environment
        packer_env = os.environ.copy()
        # This allow to speed up the typing of the preseed command line
        packer_env["PACKER_KEY_INTERVAL"] = "10ms"
        # This will add log output for packer run    
        packer_env["PACKER_LOG"] = "1" if log else "0"

        # Define the template hierarchy for the selected templates
        config_hierarchy = self._build_config_hierarchy(selected_templates)

        # Save the current directory
        current_dir = os.getcwd()

        # Loop over the template hierarchy and run packer
        for template in config_hierarchy:

            # cd to the the template directory
209
            current_template_dir = os.path.join(self._templates_base_dir,template)
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227

            build_dir = os.path.join(current_template_dir,"builds",template)

            if os.path.exists(build_dir):
                print("An image already exists for {} template. This image will be used.".format(template))
                continue

            print("Building image for {}:".format(template))

            os.chdir(current_template_dir)

            # Run packer upon the manifest.json file
            manifest_json = os.path.join(current_template_dir,"manifest.json")
            subprocess.Popen(["packer","build",manifest_json], env=packer_env)

        # cd back to the current dir
        os.chdir(current_dir)
                    
228
    def build(self, selected_templates : Optional[List[str]] = None, **kwargs):
229 230 231 232
        """Build packer on the generated manifest.json files.

        Parameters
        ----------
eric pellegrini's avatar
eric pellegrini committed
233 234
        selected_templates: list, optional
            List of strings corresponding to the packer templates to build.
235 236 237 238 239

        run: bool, optional
            If True packer will be run from the generated manifest.json files.
        """

240 241 242 243
        templates = self.get_templates_selection(selected_templates)

        if not templates:
            raise RuntimeError("Invalid or empty template selection")
244 245 246 247 248 249 250 251 252 253 254 255 256 257

        # Loop over the selected templates
        for template_name in templates:

            template = self._build_template(template_name)
            template_node = self.get_template(template_name)

            # Fetch the parent template if any
            parent_template = template_node.get("extends", None)
            if parent_template is not None:
                parent_template = self._build_template(parent_template)
                template.set_parent(parent_template)

            # Dump the template
258 259
            output_file = os.path.join(self._templates_base_dir,template_name,"manifest.json")
            template.dump(output_file,**kwargs)
260