Commit 2b05f079 authored by Eric Pellegrini's avatar Eric Pellegrini

fixed bug in manifest files

fixed runtime errors when running packer
parent 52f3cf8e
......@@ -16,7 +16,10 @@ from typing import List
from packme import Packman
def clean_templates_dir(templates_base_dir : str):
class NotSudo(Exception):
pass
def clean_templates_dir(templates_base_dir: str):
"""Removes all manifest.json files, packer_cache and builds directories found in
templates directory
"""
......@@ -34,11 +37,15 @@ def clean_templates_dir(templates_base_dir : str):
shutil.rmtree(os.path.join(template_dir,"builds"))
except FileNotFoundError:
pass
except PermissionError:
raise NotSudo("Only sudoers can remove builds directory")
try:
shutil.rmtree(os.path.join(template_dir,"packer_cache"))
except FileNotFoundError:
pass
except PermissionError:
raise NotSudo("Only sudoers can remove packer_cache directory")
def parse_args() -> argparse.Namespace:
"""Returns the argument namespace after command-line parsing.
......@@ -46,10 +53,12 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="This is packme: a program for generating and running packer configuration(s)")
parser.add_argument("--log", "-l", dest="log", action="store_true", help="log packer output")
parser.add_argument("--key-rate", "-k", dest="key_rate", default="10", help="rate for automatic key entering (in ms)")
parser.add_argument("--clean", "-c", dest="clean", action="store_true", help="clean up templates directories")
parser.add_argument("--debug", "-d", dest="debug", action="store_true", help="debug mode (keep temporary files)")
parser.add_argument("--run", "-r", dest="run", action="store_true", help="run packer after json manifest(s) generation")
parser.add_argument("--templates-base-dir", "-t", dest="templates_base_dir", required=True, help="Templates base directory")
parser.add_argument("--packages-base-dir", "-p", dest="packages_base_dir", help="Packages base directory")
parser.add_argument("--selected-templates", "-s", dest="selected_templates", nargs="*", default=["*"], help="run packman on selected template(s)")
parser.add_argument("--input", "-i", dest="input_file", help="YAML input file")
args = parser.parse_args()
......@@ -63,10 +72,12 @@ if __name__ == "__main__":
input_file : str = args.input_file
run : bool = args.run
templates_base_dir : str = args.templates_base_dir
packages_base_dir : str = args.packages_base_dir
selected_templates : List[str] = args.selected_templates
clean : bool = args.clean
debug : bool= args.debug
log : bool = args.log
key_rate : int = int(args.key_rate)
# If --clean option is set, cleanup the templates dir (builds, packer_cache, manifest.json files and directories)
if clean:
......@@ -75,12 +86,12 @@ if __name__ == "__main__":
if not input_file:
sys.exit(0)
t = Packman.Packman(input_file, templates_base_dir)
t = Packman.Packman(input_file, templates_base_dir, packages_base_dir)
t.build(selected_templates=selected_templates, indent=4, separators=(',', ': '))
if run:
t.run(selected_templates=selected_templates, log=log)
t.run(selected_templates=selected_templates, log = log, key_rate=key_rate)
# If --debug option is set, keep temporary files and directories in templates dir (builds, packer_cache, manifest.json)
if not debug:
......
......@@ -18,11 +18,14 @@ class PackerTemplate:
- processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
"""
def __init__(self, name: str, yaml_node: dict, packages: List[str]) -> None:
def __init__(self, name: str, yaml_node: dict, packages: List[str], packages_base_dir: str, templates_base_dir: str) -> None:
"""Constructor
"""
self._name = name
self._packages_base_dir = packages_base_dir
self._templates_base_dir = templates_base_dir
self._parameters = yaml_node.get("parameters", {})
......@@ -37,20 +40,45 @@ class PackerTemplate:
# Fetch the 'builders' node from the packer node
self._builders = packer_node.get("builders", [])
for builder in self._builders:
self._update_builder(builder)
# Fetch the 'provisioners' node from the packer node
self._provisioners = packer_node.get("provisioners", [])
# Case of the file based provisioner, update the source path
for provisioner in self._provisioners:
self._update_provisioner(provisioner,os.path.join(self._templates_base_dir,self._name))
# Fetch the 'postprocessors' node from the packer node
self._postprocessors = packer_node.get("postprocessors", [])
bin_dir = os.path.realpath(os.path.dirname(__file__))
root_dir = os.path.dirname(bin_dir)
self._packages_root_dir = os.path.join(root_dir,"packages")
self._templates_dir = os.path.join(root_dir,"templates")
self._load_packages(packages)
def _update_builder(self, builder):
"""Update a builder.
"""
builder["output_directory"] = "builds"
if not os.path.isabs(builder["output_directory"]):
builder["output_directory"] = os.path.join(self._templates_base_dir,self._name,builder["output_directory"])
builder["http_directory"] = "http"
if not os.path.isabs(builder["http_directory"]):
builder["http_directory"] = os.path.join(self._templates_base_dir,self._name,builder["http_directory"])
# For convenience set the vm name with a fixed standardized value
builder["vm_name"] = "{}-{}".format(self._name,builder["type"])
def _update_provisioner(self, provisioner, base_dir):
if not provisioner["type"] in ["file","shell"]:
return
for k,v in provisioner.items():
if k in ["source","script"] and not os.path.isabs(provisioner[k]):
provisioner[k] = os.path.join(base_dir,provisioner[k])
@property
def builders(self) -> list:
"""Returns the list of packer builders of this :class:`PackerTemplate`.
......@@ -128,11 +156,9 @@ class PackerTemplate:
parent_builder = {}
parent_builder["name"] = builder_name
parent_builder["type"] = builder["type"]
parent_builder["vm_name"] = self._name
parent_builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
parent_builder["iso_url"] = os.path.join(builder["output_directory"],"{}-{}".format(parent_template.name, builder_name))
parent_builder["iso_checksum_type"] = "none"
parent_builder["iso_checksum_url"] = "none"
parent_builder["output_directory"] = os.path.join(self._templates_dir, self._name)
self._builders.insert(0,parent_builder)
# If the builder is also defined in the child config, use the child config one and specify the image dependency
......@@ -142,11 +168,9 @@ class PackerTemplate:
if builder is None:
continue
builder["vm_name"] = self._name
builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
builder["iso_url"] = os.path.join(self._templates_base_dir,parent_template.name,"builds","{}-{}".format(parent_template.name, builder_name))
builder["iso_checksum_type"] = "none"
builder["iso_checksum_url"] = "none"
builder["output_directory"] = os.path.join(self._templates_dir,self._name)
def _load_packages(self, packages: List[str]):
"""Load the non-standard package YAML file and append them as provisioners of this :class:`PackerTemplate`.
......@@ -159,12 +183,12 @@ class PackerTemplate:
# If *" is in the list, fetch all the packages
if "*" in packages:
packages_dir = glob.glob(os.path.join(self._packages_root_dir,"*"))
packages_dir = glob.glob(os.path.join(self._packages_base_dir,"*"))
# Otherwise just fetch the selected ones
else:
packages_dir = []
for package in packages:
package_dir = os.path.join(self._packages_root_dir,package)
package_dir = os.path.join(self._packages_base_dir,package)
if os.path.exists(package_dir) and os.path.isdir(package_dir):
packages_dir.append(package_dir)
......@@ -186,10 +210,7 @@ class PackerTemplate:
# Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
for provisioner in manifest_data:
if provisioner["type"] in ["file"]:
provisioner["source"] = os.path.join(package_dir,provisioner["source"])
elif provisioner["type"] in ["shell"]:
provisioner["script"] = os.path.join(package_dir,provisioner["script"])
self._update_provisioner(provisioner,package_dir,self._name)
# Extend the current provisioners list with the ones of the selected packages
self._provisioners.extend(manifest_data)
......
......@@ -12,7 +12,7 @@ class Packman:
"""This class implements the Packman engine for generating packer template json files and run packer optionally .
"""
def __init__(self, input_file: str, templates_base_dir: str) -> None:
def __init__(self, input_file: str, templates_base_dir: Optional[str] = None, packages_base_dir: Optional[str] = None) -> None:
"""Constructor.
Parameters
......@@ -22,7 +22,10 @@ class Packman:
"""
# The base directory for templates
self._templates_base_dir = os.path.abspath(templates_base_dir)
self._templates_base_dir = os.path.abspath(templates_base_dir) if templates_base_dir is not None else os.path.join(os.getcwd(),"templates")
# The base directory for packages
self._packages_base_dir = os.path.abspath(packages_base_dir) if packages_base_dir is not None else os.path.join(os.getcwd(),"packages")
with open(input_file, "r") as fin:
data = yaml.safe_load(fin)
......@@ -110,7 +113,7 @@ class Packman:
packages = template_node.get("packages", [])
# Build a PackerTemplate object that can be dumped to a manifest.json file
template = PackerTemplate(template_name, manifest_data, packages)
template = PackerTemplate(template_name, manifest_data, packages, self._packages_base_dir, self._templates_base_dir)
return template
......@@ -176,7 +179,7 @@ class Packman:
return config_hierarchy
def run(self, selected_templates : Optional[List[str]] = None, log : Optional[bool] = False):
def run(self, selected_templates : Optional[List[str]] = None, log : Optional[bool] = False, key_rate: Optional[int] = 10):
"""Run packer on the generated manifest.json files.
Parameters
......@@ -189,10 +192,13 @@ class Packman:
if shutil.which("packer") is None:
raise FileNotFoundError("packer could not be found.")
if key_rate < 10:
raise ValueError("The key rate is too low. This will trigger input error.")
# Set env variables for packer run environment
packer_env = os.environ.copy()
# This allow to speed up the typing of the preseed command line
packer_env["PACKER_KEY_INTERVAL"] = "10ms"
packer_env["PACKER_KEY_INTERVAL"] = "{}ms".format(key_rate)
# This will add log output for packer run
packer_env["PACKER_LOG"] = "1" if log else "0"
......@@ -208,19 +214,18 @@ class Packman:
# cd to the the template directory
current_template_dir = os.path.join(self._templates_base_dir,template)
build_dir = os.path.join(current_template_dir,"builds",template)
build_dir = os.path.join(current_template_dir,"builds")
if os.path.exists(build_dir):
print("An image already exists for {} template. This image will be used.".format(template))
continue
print("Building image for {}:".format(template))
raise IOError("The 'builds' directory already exists. Please remove it before running packer.")
else:
print("Building image for {}:".format(template))
os.chdir(current_template_dir)
# Run packer upon the manifest.json file
manifest_json = os.path.join(current_template_dir,"manifest.json")
subprocess.Popen(["packer","build",manifest_json], env=packer_env)
subprocess.call(["packer","build",manifest_json], env=packer_env)
# cd back to the current dir
os.chdir(current_dir)
......
......@@ -52,9 +52,5 @@ packer:
source: system/etc/default/autofs
destination: /etc/default/autofs
- type: file
source: system/etc/ntp.conf
destination: /etc/ntp.conf
- type: shell
inline: ["chmod 600 /etc/sssd/sssd.conf"]
......@@ -32,10 +32,9 @@ packer:
iso_url: "{{ parameters.ubuntu_mirror }}/{{ parameters.ubuntu_codename }}-updates/main/installer-amd64/current/images/netboot/mini.iso"
ssh_username: "{{ parameters.ssh_username }}"
ssh_password: "{{ environment.root_password }}"
ssh_wait_timeout: 60m
ssh_timeout: 60m
accelerator: kvm
headless: "{{ parameters.headless }}"
output_directory: "{{ parameters.build_output_directory }}"
shutdown_command: "shutdown -P now"
qemuargs:
- - "-m"
......@@ -53,7 +52,6 @@ packer:
- "netcfg/get_nameservers={{ parameters.dns_servers }} "
- "netcfg/hostname={{ parameters.vm_name }} "
- "mirror/http/proxy={{ parameters.proxy}} "
- "clock-setup/ntp-server={{ parameters.ntp_servers }} "
- "passwd/user-fullname={{ parameters.user }} "
- "passwd/username={{ parameters.user_fullname }} "
- "passwd/user-password= {{ environment.user_password }} "
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment