Commit 52c7f45b authored by eric pellegrini's avatar eric pellegrini
Browse files

add src and scripts for packe

parent 49cef1dc
#!/usr/bin/env python3
"""
packme is a python3 application for generating and running packer templates.
It works by generating packer manifest.json files out of manifest.yml files which contain the packer settings for the image to build.
"""
import argparse
import glob
import os
import shutil
import sys
from packman import Packman
def clean_templates_dir(templates_dir):
"""Removes all manifest.json files, packer_cache and builds directories found in
templates directory
"""
for template_dir in templates_dir:
try:
os.remove(os.path.join(template_dir,"manifest.json"))
except FileNotFoundError:
pass
try:
shutil.rmtree(os.path.join(template_dir,"builds"))
except FileNotFoundError:
pass
try:
shutil.rmtree(os.path.join(template_dir,"packer_cache"))
except FileNotFoundError:
pass
def parse_args():
"""Setup and run the argument parser.
"""
parser = argparse.ArgumentParser(description="This is packme: a program for generating and running packer configuration(s)")
parser.add_argument("--log", "-l", dest="log", action="store_true", help="log packer output")
parser.add_argument("--clean", "-c", dest="clean", action="store_true", help="clean up templates directories")
parser.add_argument("--debug", "-d", dest="debug", action="store_true", help="debug mode (keep temporary files)")
parser.add_argument("--run", "-r", dest="run", action="store_true", help="run packer after json manifest(s) generation")
parser.add_argument("--templates-dir", "-t", dest="templates_dir", required=True, help="Templates base directory")
parser.add_argument("--selected-templates", "-s", dest="selected_templates", nargs="*", default=["*"], help="run packman on selected template(s)")
parser.add_argument("--input", "-i", dest="input_file", help="YAML input file")
args = parser.parse_args()
return args
if __name__ == "__main__":
# Parse command line arguments and fetch their values
args = parse_args()
input_file = args.input_file
run = args.run
templates_dir = args.templates_dir
selected_templates = args.selected_templates
clean = args.clean
debug = args.debug
log = args.log
# If --clean option is set, cleanup the templates dir (builds, packer_cache, manifest.json files and directories)
if clean:
clean_templates_dir(templates_dir)
if not input_file:
sys.exit(0)
t = Packman.Packman(input_file, templates_dir)
t.build(selected_templates=selected_templates, indent=4, separators=(',', ': '))
if run:
t.run(selected_templates=selected_templates, log=log)
# If --debug option is set, keep temporary files and directories in templates dir (builds, packer_cache, manifest.json)
if not debug:
clean_templates_dir(templates_dir)
Metadata-Version: 2.1
Name: packman
Version: 0.0.0
Summary: Heler app for building and running packer templates
Home-page: https://code.ill.fr/si/packer
Author: Eric Pellegrini
Author-email: pellegrini@ill.fr
Maintainer: Eric Pellegrini
Maintainer-email: pellegrini@ill.fr
License: MIT
Description: Overview
========
**packman** is a python3 application for generating and running packer templates out of YAML configuration file.
Prerequesites
=============
- python3 + pip
- packer application must be installed
Installation
============
see [here](https://code.ill.fr/si/packer) for complementary info
- `cd` to the directory where lies the `setup.py` file
- pip3 install --user .
Platform: Unix
Platform: Windows
Classifier: Programming Language :: Python :: 3
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Description-Content-Type: text/markdown
README.rst
setup.py
scripts/packme
src/packman/PackerTemplate.py
src/packman/Packman.py
src/packman/__init__.py
src/packman/__pkginfo__.py
src/packman.egg-info/PKG-INFO
src/packman.egg-info/SOURCES.txt
src/packman.egg-info/dependency_links.txt
src/packman.egg-info/requires.txt
src/packman.egg-info/top_level.txt
\ No newline at end of file
#!/usr/bin/env python3
#################################################################
# This apps allows the creation of packer json file out of
# YAML template
# One of its main feature is to handle template inheritance
# which is not covered yet natively by packer
# To run the application:
# env_var_1 env_var_2 ... env_var_n rackman.py template.yml
# where env_var_1 ... env_var_n are environment variables
# passed at the command line level and template.yml is the YAML
# which contain the packer configuration
#################################################################
import collections
import glob
import jinja2
import json
import os
import pprint
import yaml
class PackerTemplate:
"""This class implements one packer template.
Basically, a packer template is made of four sections:
- description: a string that explains the purpose of the template
- variables: a dictionary of variables used in the template through jinja2 mechanism
- builders: a list of dictionaries where each dictionary defines the type of image that will be built
- provisioners: a list of dictionaries where each dictionary defines actions used to configure the image
- processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
"""
def __init__(self, name, yaml_node, packages):
self._name = name
self._parameters = yaml_node.get("parameters", {})
# Fetch the 'packer' node
packer_node = yaml_node.get("packer", {})
# Fetch the 'description' node from the packer node
self._description = packer_node.get("description", "No description provided")
# Fetch the '_variables' node from the packer node
self._variables = packer_node.get("variables", {})
# Fetch the 'builders' node from the packer node
self._builders = packer_node.get("builders", [])
# Fetch the 'provisioners' node from the packer node
self._provisioners = packer_node.get("provisioners", [])
# Fetch the 'postprocessors' node from the packer node
self._postprocessors = packer_node.get("postprocessors", [])
bin_dir = os.path.realpath(os.path.dirname(__file__))
root_dir = os.path.dirname(bin_dir)
self._packages_root_dir = os.path.join(root_dir,"packages")
self._templates_dir = os.path.join(root_dir,"templates")
self._load_packages(packages)
@property
def builders(self):
"""Property for _builders attribute.
Returns
-------
list of packer builders
The list of packer builders for this PackerTemplate.
"""
return self._builders
@property
def description(self):
"""Property for _description attribute.
Returns
-------
str
The description of this PackerTemplate.
"""
return self._description
@property
def name(self):
"""Property for _name attribute.
Returns
-------
str
The name of this PackerTemplate.
"""
return self._name
@property
def parameters(self):
"""Property for _parameters attribute.
Returns
-------
dict
The parameters that will be used by Jinja 2 when creating the template json file.
"""
return self._parameters
@property
def postprocessors(self):
"""Property for _postprocessors attribute.
Returns
-------
list of packer postprocessors
The list of packer postprocessors for this PackerTemplate.
"""
return self._postprocessors
@property
def provisioners(self):
"""Property for _provisioners attribute.
Returns
-------
list of packer provisioners
The list of packer provisioners for this PackerTemplate.
"""
return self._provisioners
@property
def variables(self):
"""Property for _variables attribute.
Returns
-------
list of packer variables
The list of packer variables for this PackerTemplate.
"""
return self._variables
def set_parent(self, parent_template):
"""Set the parent template to this PackerTemplate.
This defines a relationship for future packer run in the sense that the child template will start directly from the image of
its parent template.
Parameters
----------
parent_template: :obj: `PackerTemplate`
The PackerTemplate of the parent template to connect the child template with.
"""
# List of the builder names for the child template
child_builder_names = [builder["name"] for builder in self._builders]
# Loop over the builder of the parent template
for builder in parent_template.builders:
builder_name = builder["name"]
# If this is a builder specific to the parent config, copy it in the child config and set it with an image dependency
if builder_name not in child_builder_names:
parent_builder = {}
parent_builder["name"] = builder_name
parent_builder["type"] = builder["type"]
parent_builder["vm_name"] = self._name
parent_builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
parent_builder["iso_checksum_type"] = "none"
parent_builder["iso_checksum_url"] = "none"
parent_builder["output_directory"] = os.path.join(self._templates_dir,self._name)
self._builders.insert(0,parent_builder)
# If the builder is also defined in the child config, use the child config one and specify the image dependency
else:
builder = next((b for b in self._builders if b["name"] == builder_name),None)
if builder is None:
continue
builder["vm_name"] = self._name
builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
builder["iso_checksum_type"] = "none"
builder["iso_checksum_url"] = "none"
builder["output_directory"] = os.path.join(self._templates_dir,self._name)
def _load_packages(self, packages):
"""Load the non-standard package YAML file and append them as provisioners of this PackerTemplate.
Parameters
----------
list of str
The list of the packages to append.
"""
# If *" is in the list, fetch all the packages
if "*" in packages:
packages_dir = glob.glob(os.path.join(self._packages_root_dir,"*"))
# Otherwise just fetch the selected ones
else:
packages_dir = []
for package in packages:
package_dir = os.path.join(self._packages_dir,package)
if os.path.exists(package_dir) and os.path.isdir(package_dir):
packages_dir.append(package_dir)
# Loop over the packages directories
for package_dir in packages_dir:
# Build the path to the package manifest file (YAML)
manifest_file = os.path.join(package_dir,"manifest.yml")
# Open and load the provisioners list from the manifest file
try:
fin = open(manifest_file, "r")
# If the manifest does not exist, the provisioners list is set to an empty list
except FileNotFoundError:
manifest_data = []
else:
manifest_data = yaml.safe_load(fin)
manifest_data = manifest_data["provisioners"]
# Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
for provisioner in manifest_data:
if provisioner["type"] in ["file"]:
provisioner["source"] = os.path.join(package_dir,provisioner["source"])
elif provisioner["type"] in ["shell"]:
provisioner["script"] = os.path.join(package_dir,provisioner["script"])
# Extend the current provisioners list with the ones of the selected packages
self._provisioners.extend(manifest_data)
def dump(self, output_file, **kwargs):
"""Dump this PackerTemplate to a file.
Parameters
----------
output_file: str
The path to the output json file for this PackerTemplate.
"""
# Get the basename and the ext of the output_file
basename, ext = os.path.splitext(output_file)
# If the ext is different from .json set it to .json
if ext != ".json":
ext = ".json"
# Reformat the output_file
output_file = "{}{}".format(basename, ext)
# This will be the node to be dumped
node = {}
node["description"] = self._description
node["variables"] = self._variables
node["builders"] = self._builders
node["provisioners"] = self._provisioners
node["post-processors"] = self._postprocessors
# Render the jinja2 templates with the parameters dictionary provided in the template file and the available environment variables
jinja_template = jinja2.Template(repr(node))
s = jinja_template.render(parameters=self._parameters, environment=os.environ)
# Dump to the output file
with open(output_file, "w") as fout:
json.dump(yaml.safe_load(s), fout, **kwargs)
def __str__(self):
"""Returns fancy output for this PackerTemplate.
Returns
-------
str
The string that will result from a str call to a PackerTemplate object.
"""
d = collections.OrderedDict()
d["description"] = self._description
d["variables"] = self._variables
d["builders"] = self._builders
d["provisioners"] = self._provisioners
d["post-processors"] = self._postprocessors
return pprint.pformat(d)
#!/usr/bin/env python3
#################################################################
# This apps allows the creation of packer json file out of
# YAML template
# One of its main feature is to handle template inheritance
# which is not covered yet natively by packer
# To run the application:
# env_var_1 env_var_2 ... env_var_n rackman.py template.yml
# where env_var_1 ... env_var_n are environment variables
# passed at the command line level and template.yml is the YAML
# which contain the packer configuration
#################################################################
import collections
import os
import subprocess
import yaml
from .PackerTemplate import PackerTemplate
class Packman:
"""This class implements the Packman engine for generating packer template json files and run packer optionally .
"""
def __init__(self, input_file, templates_dir):
"""Constructor.
Parameters
----------
input_file: str
Path to the Packman input file.
"""
# The base directory for templates
self._templates_dir = os.path.abspath(templates_dir)
with open(input_file, "r") as fin:
data = yaml.safe_load(fin)
# The input file must ne a YAML file which declares a 'templates' dictionary
if "templates" not in data:
raise IOError("Invalid YAML file: must contains 'templates' tag")
self._templates = data["templates"]
def get_template(self, template_name):
"""Return the YAML contents of a given template.
Parameters
----------
template_name: str
The name of the template to fetch.
Returns
-------
:obj: `PackerTemplate`
The YAML contents of the template to be fetched.
"""
return self._templates[template_name] if isinstance(self._templates[template_name],dict) else {}
def _build_template(self, template_name):
"""Build a PackerTemplate object from a template name.
Parameters
----------
template_name: str
The name of the template to build.
Returns
-------
:obj: `PackerTemplate`
The template object used by packman to build the manifest.json file.
"""
# Fetch the template matching template_name key
template_node = self.get_template(template_name)
# Build the path for the template manifest file (YAML)
manifest_file = os.path.join(self._templates_dir,template_name,"manifest.yml")
# Opens the file and load its contents
try:
fin = open(manifest_file, "r")
# If the file does not exist, the manifest contents is just an empty dict
except FileNotFoundError:
manifest_data = {}
else:
manifest_data = yaml.safe_load(fin)
# Get the packages node which gives the list of non-standard applications to add to the packer process
packages = template_node.get("packages", [])
# Build a PackerTemplate object that can be dumped to a manifest.json file
template = PackerTemplate(template_name, manifest_data, packages)
return template
def _build_template_hierarchy(self, template_name, hierarchy):
"""Build a single template hierarchy.
A template can have a parent template. In that case for packer neig able to run on those templates,
the parent tenplate must have been built before.
Getting a hierarchy of templates, the first one being the ones with no parent is the goal of this method.
Parameters
----------
template_name: str
The template on which the hierarchy will be built upon.
hierarchy: list str
The template hierarchy.
This argument is just used for passing the template hierarchy across recursive calls of this method.
"""
# Fetch the template matching template_name key
template_node = self.get_template(template_name)
extends = template_node.get("extends",None)
hierarchy.append(template_name)
if extends is None:
return
else:
self._build_template_hierarchy(extends, hierarchy)
def _build_config_hierarchy(self, selected_templates=None):
"""Build the templates hierarchy.
A template can have a parent template. In that case for packer neig able to run on those templates,
the parent tenplate must have been built before.
Getting a hierarchy of templates, the first one being the ones with no parent is the goal of this method.
Parameters
----------
selected_templates: list of str
List of packer templates.
Returns
-------
List of str
Returns the hierarchy of templates from the one with no parent to the ones with parents.
"""
if selected_templates is None:
templates = set(self._templates.keys())
else:
# Filter out the image names not present in the yaml file
templates = set([template for template in selected_templates if template in self._templates])
config_hierarchy = []
for template in templates:
self._build_template_hierarchy(template, config_hierarchy)
config_hierarchy.reverse()
config_hierarchy = list(collections.OrderedDict.fromkeys(config_hierarchy))
return config_hierarchy
def run(self, selected_templates=None, log=False):
"""Run packer on the generated manifest.json files.
Parameters
----------
selected_templates: list of str, optional
List of packer templates.
"""
# Set env variables for packer run environment
packer_env = os.environ.copy()
# This allow to speed up the typing of the preseed command line
packer_env["PACKER_KEY_INTERVAL"] = "10ms"
# This will add log output for packer run
packer_env["PACKER_LOG"] = "1" if log else "0"
# Define the template hierarchy for the selected templates
config_hierarchy = self._build_config_hierarchy(selected_templates)