Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
PaNOSC
Data Analysis Services
packme
Commits
2b05f079
Commit
2b05f079
authored
Jan 21, 2020
by
Eric Pellegrini
Browse files
fixed bug in manifest files
fixed runtime errors when running packer
parent
52f3cf8e
Changes
5
Hide whitespace changes
Inline
Side-by-side
scripts/packme
View file @
2b05f079
...
...
@@ -16,7 +16,10 @@ from typing import List
from
packme
import
Packman
def
clean_templates_dir
(
templates_base_dir
:
str
):
class
NotSudo
(
Exception
):
pass
def
clean_templates_dir
(
templates_base_dir
:
str
):
"""Removes all manifest.json files, packer_cache and builds directories found in
templates directory
"""
...
...
@@ -34,11 +37,15 @@ def clean_templates_dir(templates_base_dir : str):
shutil
.
rmtree
(
os
.
path
.
join
(
template_dir
,
"builds"
))
except
FileNotFoundError
:
pass
except
PermissionError
:
raise
NotSudo
(
"Only sudoers can remove builds directory"
)
try
:
shutil
.
rmtree
(
os
.
path
.
join
(
template_dir
,
"packer_cache"
))
except
FileNotFoundError
:
pass
except
PermissionError
:
raise
NotSudo
(
"Only sudoers can remove packer_cache directory"
)
def
parse_args
()
->
argparse
.
Namespace
:
"""Returns the argument namespace after command-line parsing.
...
...
@@ -46,10 +53,12 @@ def parse_args() -> argparse.Namespace:
parser
=
argparse
.
ArgumentParser
(
description
=
"This is packme: a program for generating and running packer configuration(s)"
)
parser
.
add_argument
(
"--log"
,
"-l"
,
dest
=
"log"
,
action
=
"store_true"
,
help
=
"log packer output"
)
parser
.
add_argument
(
"--key-rate"
,
"-k"
,
dest
=
"key_rate"
,
default
=
"10"
,
help
=
"rate for automatic key entering (in ms)"
)
parser
.
add_argument
(
"--clean"
,
"-c"
,
dest
=
"clean"
,
action
=
"store_true"
,
help
=
"clean up templates directories"
)
parser
.
add_argument
(
"--debug"
,
"-d"
,
dest
=
"debug"
,
action
=
"store_true"
,
help
=
"debug mode (keep temporary files)"
)
parser
.
add_argument
(
"--run"
,
"-r"
,
dest
=
"run"
,
action
=
"store_true"
,
help
=
"run packer after json manifest(s) generation"
)
parser
.
add_argument
(
"--templates-base-dir"
,
"-t"
,
dest
=
"templates_base_dir"
,
required
=
True
,
help
=
"Templates base directory"
)
parser
.
add_argument
(
"--packages-base-dir"
,
"-p"
,
dest
=
"packages_base_dir"
,
help
=
"Packages base directory"
)
parser
.
add_argument
(
"--selected-templates"
,
"-s"
,
dest
=
"selected_templates"
,
nargs
=
"*"
,
default
=
[
"*"
],
help
=
"run packman on selected template(s)"
)
parser
.
add_argument
(
"--input"
,
"-i"
,
dest
=
"input_file"
,
help
=
"YAML input file"
)
args
=
parser
.
parse_args
()
...
...
@@ -63,10 +72,12 @@ if __name__ == "__main__":
input_file
:
str
=
args
.
input_file
run
:
bool
=
args
.
run
templates_base_dir
:
str
=
args
.
templates_base_dir
packages_base_dir
:
str
=
args
.
packages_base_dir
selected_templates
:
List
[
str
]
=
args
.
selected_templates
clean
:
bool
=
args
.
clean
debug
:
bool
=
args
.
debug
log
:
bool
=
args
.
log
key_rate
:
int
=
int
(
args
.
key_rate
)
# If --clean option is set, cleanup the templates dir (builds, packer_cache, manifest.json files and directories)
if
clean
:
...
...
@@ -75,12 +86,12 @@ if __name__ == "__main__":
if
not
input_file
:
sys
.
exit
(
0
)
t
=
Packman
.
Packman
(
input_file
,
templates_base_dir
)
t
=
Packman
.
Packman
(
input_file
,
templates_base_dir
,
packages_base_dir
)
t
.
build
(
selected_templates
=
selected_templates
,
indent
=
4
,
separators
=
(
','
,
': '
))
if
run
:
t
.
run
(
selected_templates
=
selected_templates
,
log
=
log
)
t
.
run
(
selected_templates
=
selected_templates
,
log
=
log
,
key_rate
=
key_rate
)
# If --debug option is set, keep temporary files and directories in templates dir (builds, packer_cache, manifest.json)
if
not
debug
:
...
...
src/packme/PackerTemplate.py
View file @
2b05f079
...
...
@@ -18,11 +18,14 @@ class PackerTemplate:
- processors: a list of dictionaries where each dictionary defines actions to be run after the image is built
"""
def
__init__
(
self
,
name
:
str
,
yaml_node
:
dict
,
packages
:
List
[
str
])
->
None
:
def
__init__
(
self
,
name
:
str
,
yaml_node
:
dict
,
packages
:
List
[
str
]
,
packages_base_dir
:
str
,
templates_base_dir
:
str
)
->
None
:
"""Constructor
"""
self
.
_name
=
name
self
.
_packages_base_dir
=
packages_base_dir
self
.
_templates_base_dir
=
templates_base_dir
self
.
_parameters
=
yaml_node
.
get
(
"parameters"
,
{})
...
...
@@ -37,20 +40,45 @@ class PackerTemplate:
# Fetch the 'builders' node from the packer node
self
.
_builders
=
packer_node
.
get
(
"builders"
,
[])
for
builder
in
self
.
_builders
:
self
.
_update_builder
(
builder
)
# Fetch the 'provisioners' node from the packer node
self
.
_provisioners
=
packer_node
.
get
(
"provisioners"
,
[])
# Case of the file based provisioner, update the source path
for
provisioner
in
self
.
_provisioners
:
self
.
_update_provisioner
(
provisioner
,
os
.
path
.
join
(
self
.
_templates_base_dir
,
self
.
_name
))
# Fetch the 'postprocessors' node from the packer node
self
.
_postprocessors
=
packer_node
.
get
(
"postprocessors"
,
[])
bin_dir
=
os
.
path
.
realpath
(
os
.
path
.
dirname
(
__file__
))
root_dir
=
os
.
path
.
dirname
(
bin_dir
)
self
.
_packages_root_dir
=
os
.
path
.
join
(
root_dir
,
"packages"
)
self
.
_templates_dir
=
os
.
path
.
join
(
root_dir
,
"templates"
)
self
.
_load_packages
(
packages
)
def
_update_builder
(
self
,
builder
):
"""Update a builder.
"""
builder
[
"output_directory"
]
=
"builds"
if
not
os
.
path
.
isabs
(
builder
[
"output_directory"
]):
builder
[
"output_directory"
]
=
os
.
path
.
join
(
self
.
_templates_base_dir
,
self
.
_name
,
builder
[
"output_directory"
])
builder
[
"http_directory"
]
=
"http"
if
not
os
.
path
.
isabs
(
builder
[
"http_directory"
]):
builder
[
"http_directory"
]
=
os
.
path
.
join
(
self
.
_templates_base_dir
,
self
.
_name
,
builder
[
"http_directory"
])
# For convenience set the vm name with a fixed standardized value
builder
[
"vm_name"
]
=
"{}-{}"
.
format
(
self
.
_name
,
builder
[
"type"
])
def
_update_provisioner
(
self
,
provisioner
,
base_dir
):
if
not
provisioner
[
"type"
]
in
[
"file"
,
"shell"
]:
return
for
k
,
v
in
provisioner
.
items
():
if
k
in
[
"source"
,
"script"
]
and
not
os
.
path
.
isabs
(
provisioner
[
k
]):
provisioner
[
k
]
=
os
.
path
.
join
(
base_dir
,
provisioner
[
k
])
@
property
def
builders
(
self
)
->
list
:
"""Returns the list of packer builders of this :class:`PackerTemplate`.
...
...
@@ -128,11 +156,9 @@ class PackerTemplate:
parent_builder
=
{}
parent_builder
[
"name"
]
=
builder_name
parent_builder
[
"type"
]
=
builder
[
"type"
]
parent_builder
[
"vm_name"
]
=
self
.
_name
parent_builder
[
"iso_url"
]
=
"./builds/{}-{}"
.
format
(
parent_template
.
name
,
builder_name
)
parent_builder
[
"iso_url"
]
=
os
.
path
.
join
(
builder
[
"output_directory"
],
"{}-{}"
.
format
(
parent_template
.
name
,
builder_name
))
parent_builder
[
"iso_checksum_type"
]
=
"none"
parent_builder
[
"iso_checksum_url"
]
=
"none"
parent_builder
[
"output_directory"
]
=
os
.
path
.
join
(
self
.
_templates_dir
,
self
.
_name
)
self
.
_builders
.
insert
(
0
,
parent_builder
)
# If the builder is also defined in the child config, use the child config one and specify the image dependency
...
...
@@ -142,11 +168,9 @@ class PackerTemplate:
if
builder
is
None
:
continue
builder
[
"vm_name"
]
=
self
.
_name
builder
[
"iso_url"
]
=
"./builds/{}-{}"
.
format
(
parent_template
.
name
,
builder_name
)
builder
[
"iso_url"
]
=
os
.
path
.
join
(
self
.
_templates_base_dir
,
parent_template
.
name
,
"builds"
,
"{}-{}"
.
format
(
parent_template
.
name
,
builder_name
))
builder
[
"iso_checksum_type"
]
=
"none"
builder
[
"iso_checksum_url"
]
=
"none"
builder
[
"output_directory"
]
=
os
.
path
.
join
(
self
.
_templates_dir
,
self
.
_name
)
def
_load_packages
(
self
,
packages
:
List
[
str
]):
"""Load the non-standard package YAML file and append them as provisioners of this :class:`PackerTemplate`.
...
...
@@ -159,12 +183,12 @@ class PackerTemplate:
# If *" is in the list, fetch all the packages
if
"*"
in
packages
:
packages_dir
=
glob
.
glob
(
os
.
path
.
join
(
self
.
_packages_
root
_dir
,
"*"
))
packages_dir
=
glob
.
glob
(
os
.
path
.
join
(
self
.
_packages_
base
_dir
,
"*"
))
# Otherwise just fetch the selected ones
else
:
packages_dir
=
[]
for
package
in
packages
:
package_dir
=
os
.
path
.
join
(
self
.
_packages_
root
_dir
,
package
)
package_dir
=
os
.
path
.
join
(
self
.
_packages_
base
_dir
,
package
)
if
os
.
path
.
exists
(
package_dir
)
and
os
.
path
.
isdir
(
package_dir
):
packages_dir
.
append
(
package_dir
)
...
...
@@ -186,10 +210,7 @@ class PackerTemplate:
# Loop over the provisioners list and update when necessary relative paths with absolute one for packer to run correctly
for
provisioner
in
manifest_data
:
if
provisioner
[
"type"
]
in
[
"file"
]:
provisioner
[
"source"
]
=
os
.
path
.
join
(
package_dir
,
provisioner
[
"source"
])
elif
provisioner
[
"type"
]
in
[
"shell"
]:
provisioner
[
"script"
]
=
os
.
path
.
join
(
package_dir
,
provisioner
[
"script"
])
self
.
_update_provisioner
(
provisioner
,
package_dir
,
self
.
_name
)
# Extend the current provisioners list with the ones of the selected packages
self
.
_provisioners
.
extend
(
manifest_data
)
...
...
src/packme/Packman.py
View file @
2b05f079
...
...
@@ -12,7 +12,7 @@ class Packman:
"""This class implements the Packman engine for generating packer template json files and run packer optionally .
"""
def
__init__
(
self
,
input_file
:
str
,
templates_base_dir
:
str
)
->
None
:
def
__init__
(
self
,
input_file
:
str
,
templates_base_dir
:
Optional
[
str
]
=
None
,
packages_base_dir
:
Optional
[
str
]
=
None
)
->
None
:
"""Constructor.
Parameters
...
...
@@ -22,7 +22,10 @@ class Packman:
"""
# The base directory for templates
self
.
_templates_base_dir
=
os
.
path
.
abspath
(
templates_base_dir
)
self
.
_templates_base_dir
=
os
.
path
.
abspath
(
templates_base_dir
)
if
templates_base_dir
is
not
None
else
os
.
path
.
join
(
os
.
getcwd
(),
"templates"
)
# The base directory for packages
self
.
_packages_base_dir
=
os
.
path
.
abspath
(
packages_base_dir
)
if
packages_base_dir
is
not
None
else
os
.
path
.
join
(
os
.
getcwd
(),
"packages"
)
with
open
(
input_file
,
"r"
)
as
fin
:
data
=
yaml
.
safe_load
(
fin
)
...
...
@@ -110,7 +113,7 @@ class Packman:
packages
=
template_node
.
get
(
"packages"
,
[])
# Build a PackerTemplate object that can be dumped to a manifest.json file
template
=
PackerTemplate
(
template_name
,
manifest_data
,
packages
)
template
=
PackerTemplate
(
template_name
,
manifest_data
,
packages
,
self
.
_packages_base_dir
,
self
.
_templates_base_dir
)
return
template
...
...
@@ -176,7 +179,7 @@ class Packman:
return
config_hierarchy
def
run
(
self
,
selected_templates
:
Optional
[
List
[
str
]]
=
None
,
log
:
Optional
[
bool
]
=
False
):
def
run
(
self
,
selected_templates
:
Optional
[
List
[
str
]]
=
None
,
log
:
Optional
[
bool
]
=
False
,
key_rate
:
Optional
[
int
]
=
10
):
"""Run packer on the generated manifest.json files.
Parameters
...
...
@@ -189,10 +192,13 @@ class Packman:
if
shutil
.
which
(
"packer"
)
is
None
:
raise
FileNotFoundError
(
"packer could not be found."
)
if
key_rate
<
10
:
raise
ValueError
(
"The key rate is too low. This will trigger input error."
)
# Set env variables for packer run environment
packer_env
=
os
.
environ
.
copy
()
# This allow to speed up the typing of the preseed command line
packer_env
[
"PACKER_KEY_INTERVAL"
]
=
"
10
ms"
packer_env
[
"PACKER_KEY_INTERVAL"
]
=
"
{}
ms"
.
format
(
key_rate
)
# This will add log output for packer run
packer_env
[
"PACKER_LOG"
]
=
"1"
if
log
else
"0"
...
...
@@ -208,19 +214,18 @@ class Packman:
# cd to the the template directory
current_template_dir
=
os
.
path
.
join
(
self
.
_templates_base_dir
,
template
)
build_dir
=
os
.
path
.
join
(
current_template_dir
,
"builds"
,
template
)
build_dir
=
os
.
path
.
join
(
current_template_dir
,
"builds"
)
if
os
.
path
.
exists
(
build_dir
):
print
(
"An image already exists for {} template. This image will be used."
.
format
(
template
))
continue
print
(
"Building image for {}:"
.
format
(
template
))
raise
IOError
(
"The 'builds' directory already exists. Please remove it before running packer."
)
else
:
print
(
"Building image for {}:"
.
format
(
template
))
os
.
chdir
(
current_template_dir
)
# Run packer upon the manifest.json file
manifest_json
=
os
.
path
.
join
(
current_template_dir
,
"manifest.json"
)
subprocess
.
Popen
([
"packer"
,
"build"
,
manifest_json
],
env
=
packer_env
)
subprocess
.
call
([
"packer"
,
"build"
,
manifest_json
],
env
=
packer_env
)
# cd back to the current dir
os
.
chdir
(
current_dir
)
...
...
templates/base-autofs/manifest.yml
View file @
2b05f079
...
...
@@ -52,9 +52,5 @@ packer:
source
:
system/etc/default/autofs
destination
:
/etc/default/autofs
-
type
:
file
source
:
system/etc/ntp.conf
destination
:
/etc/ntp.conf
-
type
:
shell
inline
:
[
"
chmod
600
/etc/sssd/sssd.conf"
]
templates/base-ill/manifest.yml
View file @
2b05f079
...
...
@@ -32,10 +32,9 @@ packer:
iso_url
:
"
{{
parameters.ubuntu_mirror
}}/{{
parameters.ubuntu_codename
}}-updates/main/installer-amd64/current/images/netboot/mini.iso"
ssh_username
:
"
{{
parameters.ssh_username
}}"
ssh_password
:
"
{{
environment.root_password
}}"
ssh_
wait_
timeout
:
60m
ssh_timeout
:
60m
accelerator
:
kvm
headless
:
"
{{
parameters.headless
}}"
output_directory
:
"
{{
parameters.build_output_directory
}}"
shutdown_command
:
"
shutdown
-P
now"
qemuargs
:
-
-
"
-m"
...
...
@@ -53,7 +52,6 @@ packer:
-
"
netcfg/get_nameservers={{
parameters.dns_servers
}}
"
-
"
netcfg/hostname={{
parameters.vm_name
}}
"
-
"
mirror/http/proxy={{
parameters.proxy}}
"
-
"
clock-setup/ntp-server={{
parameters.ntp_servers
}}
"
-
"
passwd/user-fullname={{
parameters.user
}}
"
-
"
passwd/username={{
parameters.user_fullname
}}
"
-
"
passwd/user-password=
{{
environment.user_password
}}
"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment