PackerTemplate.py 11.4 KB
Newer Older
1 2 3
import collections
import glob
import jinja2
4
import jinja2.meta
5 6 7
import json
import os
import pprint
8
import sys
9 10
import yaml

11
from typing import Dict, List
12

13 14 15 16 17 18 19 20 21 22
class PackerTemplate:
    """This class implements one packer template.
    Basically, a packer template is made of four sections:
    - description: a string that explains the purpose of the template
    - variables: a dictionary of variables used in the template through jinja2 mechanism
    - builders: a list of dictionaries where each dictionary defines the type of image that will be built
    - provisioners: a list of dictionaries where each dictionary defines actions used to configure the image
    - processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
    """

Eric Pellegrini's avatar
Eric Pellegrini committed
23
    def __init__(self, name: str, yaml_node: dict, packages: List[str], packages_base_dir: str, templates_base_dir: str) -> None:
24 25
        """Constructor
        """
26

27
        self._name = name  
Eric Pellegrini's avatar
Eric Pellegrini committed
28 29 30
        
        self._packages_base_dir = packages_base_dir
        self._templates_base_dir = templates_base_dir
31

32 33 34
        environment_file = os.path.join(self._templates_base_dir,self._name,"environment.yml")
        self._read_environment_file(environment_file)
        self._environment.update(os.environ)
35 36 37 38 39 40 41 42 43 44 45 46

        # Fetch the 'packer' node
        packer_node = yaml_node.get("packer", {})

        # Fetch the 'description' node from the packer node
        self._description = packer_node.get("description", "No description provided")

        # Fetch the '_variables' node from the packer node
        self._variables = packer_node.get("variables", {})

        # Fetch the 'builders' node from the packer node
        self._builders = packer_node.get("builders", [])
Eric Pellegrini's avatar
Eric Pellegrini committed
47 48
        for builder in self._builders:
            self._update_builder(builder)
49 50 51 52

        # Fetch the 'provisioners' node from the packer node
        self._provisioners = packer_node.get("provisioners", [])

Eric Pellegrini's avatar
Eric Pellegrini committed
53 54 55 56
        # Case of the file based provisioner, update the source path
        for provisioner in self._provisioners:
            self._update_provisioner(provisioner,os.path.join(self._templates_base_dir,self._name))

57 58 59 60 61
        # Fetch the 'postprocessors' node from the packer node
        self._postprocessors = packer_node.get("postprocessors", [])

        self._load_packages(packages)

62 63 64 65 66 67 68 69 70 71 72 73 74
    def _read_environment_file(self, environment_file):
        """Read YAML environemnt file and populate os.environ

        Parameters
        ----------
        environment_file: str
            Path to the environment file.
        """

        if not os.path.exists(environment_file):
            return

        self._environment = {}
75 76
        # The vm_name is set as part of the jinja environment
        self._environment["vm_name"] = self._name
77 78 79 80 81 82

        with open(environment_file, "r") as fin:
            env = yaml.safe_load(fin)
            for k,v in env.get("environment",{}).items():
                self._environment[k] = str(v)

83 84 85 86 87 88 89
    def _update_builder(self, builder: Dict[str,str]):
        """Update some fields of a builder.

        Parameters
        ----------
        builder: dict
            The builder to update.
Eric Pellegrini's avatar
Eric Pellegrini committed
90 91 92 93 94 95 96 97 98 99 100 101 102
        """

        builder["output_directory"] = "builds"
        if not os.path.isabs(builder["output_directory"]):
            builder["output_directory"] = os.path.join(self._templates_base_dir,self._name,builder["output_directory"])

        builder["http_directory"] = "http"
        if not os.path.isabs(builder["http_directory"]):
            builder["http_directory"] = os.path.join(self._templates_base_dir,self._name,builder["http_directory"])

        # For convenience set the vm name with a fixed standardized value
        builder["vm_name"] = "{}-{}".format(self._name,builder["type"])

103 104 105 106 107 108 109 110 111 112 113
    def _update_provisioner(self, provisioner : Dict[str,str], base_dir : str):
        """Update some fields of a provisioner.

        Parameters
        ----------
        provisioner: dict
            The provisioner to update.

        base_dir: str
            The base directory for this provisioner.
        """
Eric Pellegrini's avatar
Eric Pellegrini committed
114 115 116 117 118 119 120 121

        if not provisioner["type"] in ["file","shell"]:
            return

        for k,v in provisioner.items():
            if k in ["source","script"] and not os.path.isabs(provisioner[k]):
                provisioner[k] = os.path.join(base_dir,provisioner[k])

122
    @property
123 124
    def builders(self) -> list:
        """Returns the list of packer builders of this :class:`PackerTemplate`.
125 126 127 128 129
        """

        return self._builders

    @property
130 131
    def description(self) -> str:
        """Returns the description of this :class:`PackerTemplate`.
132 133 134 135 136
        """

        return self._description

    @property
137 138
    def name(self) -> str:
        """Returns the name of this :class:`PackerTemplate`.
139 140 141 142 143
        """

        return self._name

    @property
144 145
    def postprocessors(self) -> list:
        """Returns the postprocessors of this :class:`PackerTemplate`.
146 147 148 149 150
        """

        return self._postprocessors

    @property
151 152
    def provisioners(self) -> list:
        """Returns the provisioners of this :class:`PackerTemplate`.
153 154 155 156 157
        """

        return self._provisioners

    @property
158 159
    def variables(self) -> dict:
        """Returns the variables of this :class:`PackerTemplate`.
160 161 162 163
        """

        return self._variables

164 165
    def set_parent(self, parent_template: "PackerTemplate"):
        """Set the parent template to this :class:`PackerTemplate`.
166 167 168 169 170 171

        This defines a relationship for future packer run in the sense that the child template will start directly from the image of 
        its parent template.

        Parameters
        ----------
172 173
        parent_template: :class:`PackerTemplate`
            The :class:`PackerTemplate` of the parent template to connect the child template with.
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
            
        """

        # List of the builder names for the child template
        child_builder_names = [builder["name"] for builder in self._builders]

        # Loop over the builder of the parent template
        for builder in parent_template.builders:

            builder_name = builder["name"]

            # If this is a builder specific to the parent config, copy it in the child config and set it with an image dependency
            if builder_name not in child_builder_names:
                parent_builder = {}
                parent_builder["name"] = builder_name
                parent_builder["type"] = builder["type"]
Eric Pellegrini's avatar
Eric Pellegrini committed
190
                parent_builder["iso_url"] = os.path.join(builder["output_directory"],"{}-{}".format(parent_template.name, builder_name))
191 192
                parent_builder["iso_checksum_type"] = "none"
                parent_builder["iso_checksum_url"] = "none"
193
                parent_builder["headless"] = "{{ headless }}"
194 195 196 197 198 199 200 201 202
                self._builders.insert(0,parent_builder)

            # If the builder is also defined in the child config, use the child config one and specify the image dependency
            else:

                builder = next((b for b in self._builders if b["name"] == builder_name),None)
                if builder is None:
                    continue

Eric Pellegrini's avatar
Eric Pellegrini committed
203
                builder["iso_url"] = os.path.join(self._templates_base_dir,parent_template.name,"builds","{}-{}".format(parent_template.name, builder_name))
204 205
                builder["iso_checksum_type"] = "none"
                builder["iso_checksum_url"] = "none"
206
                builder["headless"] = "{{ headless }}"
207

208 209
    def _load_packages(self, packages: List[str]):
        """Load the non-standard package YAML file and append them as provisioners of this :class:`PackerTemplate`.
210 211 212

        Parameters
        ----------
eric pellegrini's avatar
eric pellegrini committed
213
        list
214
            The non-standard packages to append.    
215 216 217 218
        """

        # If *" is in the list, fetch all the packages
        if "*" in packages:
Eric Pellegrini's avatar
Eric Pellegrini committed
219
            packages_dir = glob.glob(os.path.join(self._packages_base_dir,"*"))
220 221 222 223
        # Otherwise just fetch the selected ones
        else:
            packages_dir = []
            for package in packages:
Eric Pellegrini's avatar
Eric Pellegrini committed
224
                package_dir = os.path.join(self._packages_base_dir,package)
225 226 227 228 229 230 231 232 233 234 235 236 237 238
                if os.path.exists(package_dir) and os.path.isdir(package_dir):
                    packages_dir.append(package_dir)

        # Loop over the packages directories
        for package_dir in packages_dir:
            
            # Build the path to the package manifest file (YAML)
            manifest_file = os.path.join(package_dir,"manifest.yml")
            
            # Open and load the provisioners list from the manifest file
            try:
              fin = open(manifest_file, "r")
            # If the manifest does not exist, the provisioners list is set to an empty list
            except FileNotFoundError:
239
                manifest_data: List = []
240
            else:
241 242
                root_node = yaml.safe_load(fin)
                manifest_data = root_node["provisioners"]
243 244 245

            # Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
            for provisioner in manifest_data:
246
                self._update_provisioner(provisioner,os.path.join(package_dir,self._name))
247 248 249 250

            # Extend the current provisioners list with the ones of the selected packages
            self._provisioners.extend(manifest_data)
            
251
    def dump(self, output_file: str, **kwargs):
252 253 254 255 256
        """Dump this PackerTemplate to a file.

        Parameters
        ----------
        output_file: str
257
            The path to the output json file for this :class:`PackerTemplate`.
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
        """

        # Get the basename and the ext of the output_file
        basename, ext = os.path.splitext(output_file)
        # If the ext is different from .json set it to .json
        if ext != ".json":
            ext = ".json"

        # Reformat the output_file
        output_file = "{}{}".format(basename, ext)

        # This will be the node to be dumped
        node = {}
        node["description"] = self._description
        node["variables"] = self._variables
        node["builders"] = self._builders
        node["provisioners"] = self._provisioners
        node["post-processors"] = self._postprocessors

277 278 279 280 281 282 283 284 285
        stringified_node = repr(node)

        # Create a jinja evironment for parsing the manifest
        env = jinja2.Environment()
        # Parse the string with jinja templates in unrendered state in order to get te set of all jinja templates
        ast = env.parse(stringified_node)
        jinja_templates = jinja2.meta.find_undeclared_variables(ast)
        # Retrieve those templates which are not declared in the environment. These are the undefined templates.
        undefined = jinja_templates.difference(self._environment.keys())
286

287
        # If there are some undefined templates, stop here.
288 289 290
        if undefined:
            print(f'The following variables are undefined: {undefined!r}')
            sys.exit(1)
291 292 293
        
        template = env.from_string(stringified_node)
        rendered = template.render(**self._environment)
294

295 296
        # Dump to the output file
        with open(output_file, "w") as fout:
297
            json.dump(yaml.safe_load(rendered), fout, **kwargs)
298

299 300
    def __str__(self) -> str:
        """Returns the string representation for this :class:`PackerTemplate`.
301 302
        """

303
        d : collections.OrderedDict = collections.OrderedDict()
304 305 306 307 308 309 310
        d["description"] = self._description
        d["variables"] = self._variables
        d["builders"] = self._builders
        d["provisioners"] = self._provisioners
        d["post-processors"] = self._postprocessors

        return pprint.pformat(d)