fixed bug in manifest files

fixed runtime errors when running packer

scripts/packme

@@ -16,7 +16,10 @@ from typing import List
 
 from packme import Packman
 
-def clean_templates_dir(templates_base_dir : str):
+class NotSudo(Exception):
+    pass
+
+def clean_templates_dir(templates_base_dir: str):
     """Removes all manifest.json files, packer_cache and builds directories found in
     templates directory
     """
@@ -34,11 +37,15 @@ def clean_templates_dir(templates_base_dir : str):
             shutil.rmtree(os.path.join(template_dir,"builds"))
         except FileNotFoundError:
             pass
+        except PermissionError:
+            raise NotSudo("Only sudoers can remove builds directory")
 
         try:
             shutil.rmtree(os.path.join(template_dir,"packer_cache"))
         except FileNotFoundError:
             pass
+        except PermissionError:
+            raise NotSudo("Only sudoers can remove packer_cache directory")
 
 def parse_args() -> argparse.Namespace:
     """Returns the argument namespace after command-line parsing.
@@ -46,10 +53,12 @@ def parse_args() -> argparse.Namespace:
 
     parser = argparse.ArgumentParser(description="This is packme: a program for generating and running packer configuration(s)")
     parser.add_argument("--log", "-l", dest="log", action="store_true", help="log packer output")
+    parser.add_argument("--key-rate", "-k", dest="key_rate", default="10", help="rate for automatic key entering (in ms)")
     parser.add_argument("--clean", "-c", dest="clean", action="store_true", help="clean up templates directories")
     parser.add_argument("--debug", "-d", dest="debug", action="store_true", help="debug mode (keep temporary files)")
     parser.add_argument("--run", "-r", dest="run", action="store_true", help="run packer after json manifest(s) generation")
     parser.add_argument("--templates-base-dir", "-t", dest="templates_base_dir", required=True, help="Templates base directory")
+    parser.add_argument("--packages-base-dir", "-p", dest="packages_base_dir", help="Packages base directory")
     parser.add_argument("--selected-templates", "-s", dest="selected_templates", nargs="*", default=["*"], help="run packman on selected template(s)")
     parser.add_argument("--input", "-i", dest="input_file", help="YAML input file")
     args = parser.parse_args()
@@ -63,10 +72,12 @@ if __name__ == "__main__":
     input_file : str = args.input_file
     run : bool = args.run
     templates_base_dir : str = args.templates_base_dir
+    packages_base_dir : str = args.packages_base_dir
     selected_templates : List[str] = args.selected_templates
     clean : bool = args.clean
     debug : bool= args.debug
     log : bool = args.log
+    key_rate : int = int(args.key_rate)
 
     # If --clean option is set, cleanup the templates dir (builds, packer_cache, manifest.json files and directories)
     if clean:
@@ -75,12 +86,12 @@ if __name__ == "__main__":
     if not input_file:
         sys.exit(0)
 
-    t = Packman.Packman(input_file, templates_base_dir)
+    t = Packman.Packman(input_file, templates_base_dir, packages_base_dir)
 
     t.build(selected_templates=selected_templates, indent=4, separators=(',', ': '))
 
     if run:
-        t.run(selected_templates=selected_templates, log=log)
+        t.run(selected_templates=selected_templates, log = log, key_rate=key_rate)
 
     # If --debug option is set, keep temporary files and directories in templates dir (builds, packer_cache, manifest.json)
     if not debug:

src/packme/PackerTemplate.py

@@ -18,11 +18,14 @@ class PackerTemplate:
     - processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
     """
 
-    def __init__(self, name: str, yaml_node: dict, packages: List[str]) -> None:
+    def __init__(self, name: str, yaml_node: dict, packages: List[str], packages_base_dir: str, templates_base_dir: str) -> None:
         """Constructor
         """
 
         self._name = name
+        
+        self._packages_base_dir = packages_base_dir
+        self._templates_base_dir = templates_base_dir
 
         self._parameters = yaml_node.get("parameters", {})
 
@@ -37,20 +40,45 @@ class PackerTemplate:
 
         # Fetch the 'builders' node from the packer node
         self._builders = packer_node.get("builders", [])
+        for builder in self._builders:
+            self._update_builder(builder)
 
         # Fetch the 'provisioners' node from the packer node
         self._provisioners = packer_node.get("provisioners", [])
 
+        # Case of the file based provisioner, update the source path
+        for provisioner in self._provisioners:
+            self._update_provisioner(provisioner,os.path.join(self._templates_base_dir,self._name))
+
         # Fetch the 'postprocessors' node from the packer node
         self._postprocessors = packer_node.get("postprocessors", [])
 
-        bin_dir = os.path.realpath(os.path.dirname(__file__))
-        root_dir = os.path.dirname(bin_dir)
-        self._packages_root_dir = os.path.join(root_dir,"packages")
-        self._templates_dir = os.path.join(root_dir,"templates")
-
         self._load_packages(packages)
 
+    def _update_builder(self, builder):
+        """Update a builder.
+        """
+
+        builder["output_directory"] = "builds"
+        if not os.path.isabs(builder["output_directory"]):
+            builder["output_directory"] = os.path.join(self._templates_base_dir,self._name,builder["output_directory"])
+
+        builder["http_directory"] = "http"
+        if not os.path.isabs(builder["http_directory"]):
+            builder["http_directory"] = os.path.join(self._templates_base_dir,self._name,builder["http_directory"])
+
+        # For convenience set the vm name with a fixed standardized value
+        builder["vm_name"] = "{}-{}".format(self._name,builder["type"])
+
+    def _update_provisioner(self, provisioner, base_dir):
+
+        if not provisioner["type"] in ["file","shell"]:
+            return
+
+        for k,v in provisioner.items():
+            if k in ["source","script"] and not os.path.isabs(provisioner[k]):
+                provisioner[k] = os.path.join(base_dir,provisioner[k])
+
     @property
     def builders(self) -> list:
         """Returns the list of packer builders of this :class:`PackerTemplate`.
@@ -128,11 +156,9 @@ class PackerTemplate:
                 parent_builder = {}
                 parent_builder["name"] = builder_name
                 parent_builder["type"] = builder["type"]
-                parent_builder["vm_name"] = self._name
-                parent_builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
+                parent_builder["iso_url"] = os.path.join(builder["output_directory"],"{}-{}".format(parent_template.name, builder_name))
                 parent_builder["iso_checksum_type"] = "none"
                 parent_builder["iso_checksum_url"] = "none"
-                parent_builder["output_directory"] = os.path.join(self._templates_dir, self._name)
                 self._builders.insert(0,parent_builder)
 
             # If the builder is also defined in the child config, use the child config one and specify the image dependency
@@ -142,11 +168,9 @@ class PackerTemplate:
                 if builder is None:
                     continue
 
-                builder["vm_name"] = self._name
-                builder["iso_url"] = "./builds/{}-{}".format(parent_template.name, builder_name)
+                builder["iso_url"] = os.path.join(self._templates_base_dir,parent_template.name,"builds","{}-{}".format(parent_template.name, builder_name))
                 builder["iso_checksum_type"] = "none"
                 builder["iso_checksum_url"] = "none"
-                builder["output_directory"] = os.path.join(self._templates_dir,self._name)            
 
     def _load_packages(self, packages: List[str]):
         """Load the non-standard package YAML file and append them as provisioners of this :class:`PackerTemplate`.
@@ -159,12 +183,12 @@ class PackerTemplate:
 
         # If *" is in the list, fetch all the packages
         if "*" in packages:
-            packages_dir = glob.glob(os.path.join(self._packages_root_dir,"*"))
+            packages_dir = glob.glob(os.path.join(self._packages_base_dir,"*"))
         # Otherwise just fetch the selected ones
         else:
             packages_dir = []
             for package in packages:
-                package_dir = os.path.join(self._packages_root_dir,package)
+                package_dir = os.path.join(self._packages_base_dir,package)
                 if os.path.exists(package_dir) and os.path.isdir(package_dir):
                     packages_dir.append(package_dir)
 
@@ -186,10 +210,7 @@ class PackerTemplate:
 
             # Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
             for provisioner in manifest_data:
-                if provisioner["type"] in ["file"]:
-                    provisioner["source"] = os.path.join(package_dir,provisioner["source"])
-                elif provisioner["type"] in ["shell"]:
-                    provisioner["script"] = os.path.join(package_dir,provisioner["script"])
+                self._update_provisioner(provisioner,package_dir,self._name)
 
             # Extend the current provisioners list with the ones of the selected packages
             self._provisioners.extend(manifest_data)

src/packme/Packman.py

@@ -12,7 +12,7 @@ class Packman:
     """This class implements the Packman engine for generating packer template json files and run packer optionally .
     """
 
-    def __init__(self, input_file: str, templates_base_dir: str) -> None:
+    def __init__(self, input_file: str, templates_base_dir: Optional[str] = None, packages_base_dir: Optional[str] = None) -> None:
         """Constructor.
 
         Parameters
@@ -22,7 +22,10 @@ class Packman:
         """
 
         # The base directory for templates
-        self._templates_base_dir = os.path.abspath(templates_base_dir)
+        self._templates_base_dir = os.path.abspath(templates_base_dir) if templates_base_dir is not None else os.path.join(os.getcwd(),"templates")
+
+        # The base directory for packages
+        self._packages_base_dir = os.path.abspath(packages_base_dir) if packages_base_dir is not None else os.path.join(os.getcwd(),"packages")
 
         with open(input_file, "r") as fin:
             data = yaml.safe_load(fin)
@@ -110,7 +113,7 @@ class Packman:
         packages = template_node.get("packages", [])
 
         # Build a PackerTemplate object that can be dumped to a manifest.json file
-        template = PackerTemplate(template_name, manifest_data, packages)
+        template = PackerTemplate(template_name, manifest_data, packages, self._packages_base_dir, self._templates_base_dir)
 
         return template
 
@@ -176,7 +179,7 @@ class Packman:
 
         return config_hierarchy
 
-    def run(self, selected_templates : Optional[List[str]] = None, log : Optional[bool] = False):
+    def run(self, selected_templates : Optional[List[str]] = None, log : Optional[bool] = False, key_rate: Optional[int] = 10):
         """Run packer on the generated manifest.json files.
 
         Parameters
@@ -189,10 +192,13 @@ class Packman:
         if shutil.which("packer") is None:
             raise FileNotFoundError("packer could not be found.")
 
+        if key_rate < 10:
+            raise ValueError("The key rate is too low. This will trigger input error.")
+
         # Set env variables for packer run environment
         packer_env = os.environ.copy()
         # This allow to speed up the typing of the preseed command line
-        packer_env["PACKER_KEY_INTERVAL"] = "10ms"
+        packer_env["PACKER_KEY_INTERVAL"] = "{}ms".format(key_rate)
         # This will add log output for packer run    
         packer_env["PACKER_LOG"] = "1" if log else "0"
 
@@ -208,19 +214,18 @@ class Packman:
             # cd to the the template directory
             current_template_dir = os.path.join(self._templates_base_dir,template)
 
-            build_dir = os.path.join(current_template_dir,"builds",template)
+            build_dir = os.path.join(current_template_dir,"builds")
 
             if os.path.exists(build_dir):
-                print("An image already exists for {} template. This image will be used.".format(template))
-                continue
-
-            print("Building image for {}:".format(template))
+                raise IOError("The 'builds' directory already exists. Please remove it before running packer.")
+            else:
+                print("Building image for {}:".format(template))
 
             os.chdir(current_template_dir)
 
             # Run packer upon the manifest.json file
             manifest_json = os.path.join(current_template_dir,"manifest.json")
-            subprocess.Popen(["packer","build",manifest_json], env=packer_env)
+            subprocess.call(["packer","build",manifest_json], env=packer_env)
 
         # cd back to the current dir
         os.chdir(current_dir)

templates/base-autofs/manifest.yml

@@ -52,9 +52,5 @@ packer:
       source: system/etc/default/autofs
       destination: /etc/default/autofs
 
-    - type: file
-      source: system/etc/ntp.conf
-      destination: /etc/ntp.conf
-
     - type: shell
       inline: ["chmod 600 /etc/sssd/sssd.conf"]

templates/base-ill/manifest.yml

@@ -32,10 +32,9 @@ packer:
       iso_url: "{{ parameters.ubuntu_mirror }}/{{ parameters.ubuntu_codename }}-updates/main/installer-amd64/current/images/netboot/mini.iso"
       ssh_username: "{{ parameters.ssh_username }}"
       ssh_password: "{{ environment.root_password }}"
-      ssh_wait_timeout: 60m
+      ssh_timeout: 60m
       accelerator: kvm
       headless: "{{ parameters.headless }}"
-      output_directory: "{{ parameters.build_output_directory }}"
       shutdown_command: "shutdown -P now"
       qemuargs:
         - - "-m"
@@ -53,7 +52,6 @@ packer:
         - "netcfg/get_nameservers={{ parameters.dns_servers }} "
         - "netcfg/hostname={{ parameters.vm_name }} "
         - "mirror/http/proxy={{ parameters.proxy}} "
-        - "clock-setup/ntp-server={{ parameters.ntp_servers }} "
         - "passwd/user-fullname={{ parameters.user }} "
         - "passwd/username={{ parameters.user_fullname }} "
         - "passwd/user-password= {{ environment.user_password }} "