Commit 5feb1521 authored by eric pellegrini's avatar eric pellegrini

added role for installing and configuring jupyterhub

parent 5a5e4633
---
- hosts: jhub_servers
remote_user: "{{ cluster_users.1.name }}"
environment: "{{ proxy_settings }}"
tasks:
- import_role:
name: roles/jupyterhub
vars:
jupyterhub_admin: "{{ cluster_users.1.name }}"
jupyterhub_admin_group: "{{ cluster_users.1.group }}"
keycloak_url: "{{ keycloak['url'] }}"
keycloak_admin: "{{ keycloak['admin'] }}"
keycloak_admin_password: "{{ keycloak['password'] }}"
keycloak_realm_name: "{{ keycloak['realm_name'] }}"
keycloak_port: "{{ keycloak['port'] }}"
keycloak_description: "{{ keycloak['description'] }}"
Role Name
=========
A brief description of the role goes here.
Requirements
------------
Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
Role Variables
--------------
A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
Dependencies
------------
A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
Example Playbook
----------------
Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
- hosts: servers
roles:
- { role: username.rolename, x: 42 }
License
-------
BSD
Author Information
------------------
An optional section for the role authors to include contact information, or a website (HTML is not allowed).
---
# defaults file for roles/jupyterhub
\ No newline at end of file
name: visa-jupyter
channels:
- conda-forge
- defaults
dependencies:
- notebook
- jupyterlab
- jupyterhub
- oauthenticator
- configurable-http-proxy
- sudospawner
- nb_conda
- nb_conda_kernels
{
"NotebookApp":
{
"nbserver_extensions":
{
"nb_conda": true
},
"kernel_spec_manager_class": "nb_conda_kernels.CondaKernelSpecManager"
},
"CondaKernelSpecManager":
{
"name_format": "Env: {1}"
}
}
---
# handlers file for roles/jupyterhub
- name: restart visa-jupyter
systemd:
name: visa-jupyter
enabled: yes
state: restarted
become: True
galaxy_info:
author: your name
description: your description
company: your company (optional)
# If the issue tracker for your role is not on github, uncomment the
# next line and provide a value
# issue_tracker_url: http://example.com/issue/tracker
# Choose a valid license ID from https://spdx.org - some suggested licenses:
# - BSD-3-Clause (default)
# - MIT
# - GPL-2.0-or-later
# - GPL-3.0-only
# - Apache-2.0
# - CC-BY-4.0
license: license (GPL-2.0-or-later, MIT, etc)
min_ansible_version: 2.4
# If this a Container Enabled role, provide the minimum Ansible Container version.
# min_ansible_container_version:
#
# Provide a list of supported platforms, and for each platform a list of versions.
# If you don't wish to enumerate all versions for a particular platform, use 'all'.
# To view available platforms and versions (or releases), visit:
# https://galaxy.ansible.com/api/v1/platforms/
#
# platforms:
# - name: Fedora
# versions:
# - all
# - 25
# - name: SomePlatform
# versions:
# - all
# - 1.0
# - 7
# - 99.99
galaxy_tags: []
# List tags for your role here, one per line. A tag is a keyword that describes
# and categorizes the role. Users find roles by searching for tags. Be sure to
# remove the '[]' above, if you add tags to this list.
#
# NOTE: A tag is limited to a single word comprised of alphanumeric characters.
# Maximum 20 tags per role.
dependencies: []
# List your role dependencies here, one per line. Be sure to remove the '[]' above,
# if you add dependencies to this list.
\ No newline at end of file
---
# tasks file for roles/jupyterhub
- block:
- name: create jupyter lab settings directory
file:
path: "{{ conda_envs_dir }}/visa-jupyter/share/jupyter/lab/settings"
state: directory
- name: create jupyter page_config.json configuration file
template:
src: page_config.json.j2
dest: "{{ conda_envs_dir }}/visa-jupyter/share/jupyter/lab/settings/page_config.json"
- name: remove current jupyter configuration file
file:
path: "{{ conda_envs_dir }}/visa-jupyter/etc/jupyter/jupyter_notebook_config.json"
state: absent
- name: copy the jupyter config file
copy:
src: jupyter_notebook_config.json
dest: "{{ conda_envs_dir }}/visa-jupyter/etc/jupyter/"
- name: copy the ILL logo
copy:
src: ill_logo.jpg
dest: "{{ conda_envs_dir }}/visa-jupyter/share/jupyterhub/templates/"
- name: change the jupyterhub login message
lineinfile:
path: "{{ conda_envs_dir }}/visa-jupyter/share/jupyterhub/templates/login.html"
line: "Sign in with ILL user account"
regexp: "Sign in with .*"
state: present
- name: template the python script for fetching keycloak client info
template:
src: "get_client_info.py.j2"
dest: "~/get_client_info.py"
force: True
- name: run the script
command: "python3 ~/get_client_info.py {{ keycloak_admin_password }}"
register: get_client_info
- name: set facts about keycloak client id and secret
set_fact:
client_id: "{{ client_info.id }}"
client_secret: "{{ client_info.secret }}"
vars:
client_info: "{{ get_client_info.stdout | from_yaml }}"
- name: copy the jupyterhub configuration file
template:
src: jupyterhub_config.py.j2
dest: "{{ conda_envs_dir }}/visa-jupyter/etc/jupyter/jupyterhub_config.py"
force: True
vars:
conda_envs_dir: "~/miniconda3/envs"
---
# tasks file for roles/jupyterhub
- apt:
name: python3-pip
force_apt_get: yes
state: present
update_cache: yes
become: True
- pip:
name: python-keycloak
become: True
- block:
- name: copy the conda environment file for visa jupyter
copy:
src: environment_visa_jupyter.yml
dest: /tmp
- name: create visa jupyter environment
command: "{{ conda_exe }} env create -f /tmp/environment_visa_jupyter.yml --force"
- name: update jupyterlab and jupyterhub and install jupyter labextension
shell: |
source "{{ conda_root }}/etc/profile.d/conda.sh"
conda activate visa-jupyter
conda update -y jupyterlab
conda update -y jupyterhub
jupyter labextension install @jupyterlab/hub-extension
args:
executable: /bin/bash
vars:
conda_exe: "~/miniconda3/bin/conda"
conda_root: "~/miniconda3"
---
# tasks file for roles/jupyterhub
- import_tasks: install.yml
- import_tasks: configure.yml
- import_tasks: service.yml
---
- block:
- name: copy the visa-jupyter service file
template:
src: visa-jupyter.service.j2
dest: /etc/systemd/system/visa-jupyter.service
become: True
- name: copy the jupyter start script
template:
src: start_jupyterhub.sh.j2
dest: "~/start_jupyterhub.sh"
mode: u+x
notify: restart visa-jupyter
vars:
conda_root: "~/miniconda3"
conda_envs_dir: "~/miniconda3/envs"
import sys
from keycloak import KeycloakAdmin
if __name__ == "__main__":
password = sys.argv[1]
keycloak_admin = KeycloakAdmin(server_url="{{ keycloak_url }}/auth/",
username="{{ keycloak_admin }}",
password=password,
realm_name="{{ keycloak_realm_name }}",
verify=True)
client_name = "{{ ansible_hostname }}"
client_ip = "{{ ansible_default_ipv4.address }}"
client_port = "{{ keycloak_port }}"
visa_jupyter_client_id = keycloak_admin.get_client_id(client_name)
if visa_jupyter_client_id:
keycloak_admin.delete_client(visa_jupyter_client_id)
_ = keycloak_admin.create_client({"clientId": client_name,
"attributes":{"login_itheme":"ill"},
"name": client_name,
"clientAuthenticatorType":"client-secret",
"description":"{{ keycloak_description | default("") }}",
"consentRequired":True,
"publicClient": False,
"baseUrl": "/",
"protocol":"openid-connect",
"rootUrl":"http://%s:%s" % (client_ip,client_port),
"redirectUris":["https://%s/*" % client_ip,"http://%s/*" % client_ip],
"webOrigins":[],
"adminUrl": "",
"serviceAccountsEnabled": True,
"enabled": True}, skip_exists=True)
visa_jupyter_client_id = keycloak_admin.get_client_id(client_name)
client_info = {"id":client_name, "secret":keycloak_admin.get_client_secrets(visa_jupyter_client_id)["value"]}
print(client_info)
# Configuration file for jupyterhub.
import os
import json
import os
import sys
import tempfile
import urllib
from tornado import gen, web
from tornado.auth import OAuth2Mixin
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
from tornado.httputil import url_concat
from jupyterhub.auth import LocalAuthenticator
from jupyterhub.handlers import LogoutHandler
from jupyterhub.utils import url_path_join
from oauthenticator.oauth2 import OAuthLoginHandler, OAuthenticator
class KeycloakMixin(OAuth2Mixin):
_OAUTH_AUTHORIZE_URL = "{{ keycloak_url }}/auth/realms/{{ keycloak_realm_name }}/protocol/openid-connect/auth"
_OAUTH_ACCESS_TOKEN_URL = "{{ keycloak_url }}/auth/realms/{{ keycloak_realm_name }}/protocol/openid-connect/token"
_OAUTH_LOGOUT_URL = "{{ keycloak_url }}/auth/realms/{{ keycloak_realm_name }}/protocol/openid-connect/logout"
_OAUTH_USERINFO_URL = "{{ keycloak_url }}/auth/realms/{{ keycloak_realm_name }}/protocol/openid-connect/userinfo"
class KeycloakLoginHandler(OAuthLoginHandler, KeycloakMixin):
pass
class KeycloakLogoutHandler(LogoutHandler, KeycloakMixin):
def get(self):
# Get the current user and clear its login cookie
user = self.get_current_user()
if user:
self.log.info("User logged out: %s", user.name)
self.clear_login_cookie()
self.statsd.incr('logout')
# The logout will access login.ill.fr keycloak logout which will further redirect to jupyterhub login page
params = dict(redirect_uri="%s://%s%slogin" % (self.request.protocol, self.request.host,self.hub.server.base_url))
logout_url = KeycloakMixin._OAUTH_LOGOUT_URL
logout_url = url_concat(logout_url, params)
self.redirect(logout_url, permanent=False)
class KeycloakOAuthenticator(OAuthenticator, KeycloakMixin):
login_service = "Keycloak"
login_handler = KeycloakLoginHandler
def logout_url(self, base_url):
return url_path_join(base_url, 'oauth_logout')
def get_handlers(self, app):
handlers = OAuthenticator.get_handlers(self, app)
# Override the logout handler with the keycloak logout handler
handlers.extend([(r'/logout', KeycloakLogoutHandler)])
handlers.extend([(r'/oauth_logout', KeycloakLogoutHandler)])
return handlers
@gen.coroutine
def authenticate(self, handler, data=None):
code = handler.get_argument("code", False)
if not code:
raise web.HTTPError(400, "oauth callback made without a token")
http_client = AsyncHTTPClient()
params = dict(
grant_type='authorization_code',
code=code,
redirect_uri=self.get_callback_url(handler),
)
tokenUrl = KeycloakMixin._OAUTH_ACCESS_TOKEN_URL
tokenReq = HTTPRequest(tokenUrl,
method="POST",
headers={"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded;charset=utf-8"},
auth_username=self.client_id,
auth_password=self.client_secret,
body=urllib.parse.urlencode(params).encode(
'utf-8'),
)
tokenResp = yield http_client.fetch(tokenReq)
tokenResp_json = json.loads(tokenResp.body.decode('utf8', 'replace'))
access_token = tokenResp_json['access_token']
if not access_token:
raise web.HTTPError(400, "failed to get access token")
self.log.info('oauth token: %r', access_token)
userInfoUrl = KeycloakMixin._OAUTH_USERINFO_URL
userInfoReq = HTTPRequest(userInfoUrl,
method="GET",
headers={"Accept": "application/json",
"Authorization": "Bearer %s" % access_token},
)
userInfoResp = yield http_client.fetch(userInfoReq)
userInfoResp_json = json.loads(
userInfoResp.body.decode('utf8', 'replace'))
return userInfoResp_json['preferred_username']
class LocalKeycloakOAuthenticator(LocalAuthenticator, KeycloakOAuthenticator):
"""A version that mixes in local system user creation"""
pass
bin_dir = os.path.split(sys.executable)[0]
spawner = os.path.join(bin_dir, 'sudospawner')
database = os.path.join(tempfile.gettempdir(), "jupyterhub", "jupyterhub.sqlite")
cookie = os.path.join(tempfile.gettempdir(), "jupyterhub", "jupyterhub_cookie_secret")
if not os.path.isdir(os.path.dirname(database)):
os.makedirs(os.path.dirname(database))
# Keep some environment variables if needed:
for var in ("PYTHONHOME", "PYTHONPATH"):
if var in os.environ:
c.SudoSpawner.env_keep.append(var)
# Use the sudo spawner for launching the server under a user name different than root
c.JupyterHub.spawner_class = 'sudospawner.SudoSpawner'
c.JupyterHub.ip = "{{ ansible_default_ipv4.address}}"
c.JupyterHub.port = {{ keycloak_port }}
c.JupyterHub.db_url = database
c.JupyterHub.cookie_secret_file = cookie
c.SudoSpawner.sudospawner_path = spawner
c.JupyterHub.authenticator_class = KeycloakOAuthenticator
c.OAuthenticator.client_id = "{{ client_id }}"
c.OAuthenticator.client_secret = "{{ client_secret }}"
# Here are the jupyter infrastructure admins
c.Authenticator.admin_users = {"pellegrini","pinet","hall","caunt","perrin"}
c.JupyterHub.template_paths = ["{{ conda_envs_dir }}/visa-jupyter/share/jupyterhub/templates"]
c.JupyterHub.logo_file = "{{ conda_envs_dir }}/visa-jupyter/share/jupyterhub/templates/ill_logo.jpg"
c.Spawner.default_url = "/lab"
c.Spawner.cmd = ['jupyter-labhub']
{
"hub_prefix": "{{ '~' | expand_user }}"
}
#/bin/bash
source "{{ conda_root }}/etc/profile.d/conda.sh"
conda activate visa-jupyter
cd
jupyterhub -f "{{ conda_envs_dir }}/visa-jupyter/etc/jupyter/jupyterhub_config.py"
[Unit]
Description=visa-jupyter
After=syslog.target network.target
[Service]
Environment="PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"
ExecStart=/bin/bash "~/start_jupyterhub.sh"
User={{ jupyterhub_admin }}
Group={{ jupyterhub_admin_group }}
[Install]
WantedBy=multi-user.target
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment