Skip to content

Commit

Permalink
modify jupyterhub config yaml
Browse files Browse the repository at this point in the history
  • Loading branch information
wujia committed Mar 15, 2022
1 parent 9605702 commit f39a050
Showing 1 changed file with 124 additions and 104 deletions.
228 changes: 124 additions & 104 deletions charts/jupyterhub-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,25 +46,68 @@ hub:
Authenticator:
admin_users:
- admin
baseUrl: "/jupyter"
baseUrl: "/"
extraEnv:
MAGNOTE_HOST: "http://magnote.pub/"
extraConfig:
# Register Dask Gateway service and setup singleuser environment.
00-add-dask-gateway-values: |
# Set user options form and profile lists
00-set-profile-list: |
c.Spawner.debug = True
c.Spawner.notebook_dir = '/home/jovyan/work'
c.Spawner.args = ['--ResourceUseDisplay.track_cpu_percent=True',\
c.Spawner.args = ['--NotebookApp.shutdown_no_activity_timeout=7200',\
'--MappingKernelManager.cull_idle_timeout=1800',\
'--MappingKernelManager.cull_interval=300',\
'--ResourceUseDisplay.track_cpu_percent=True',\
'--ResourceUseDisplay.mem_warning_threshold=0.1']
default_env = "1C2G"
c.KubeSpawner.options_form = """
<style>
/* The profile description should not be bold, even though it is inside the <label> tag */
#kubespawner-profiles-list label p {
font-weight: normal;
}
textarea.form-control {
height: 100px;
}
</style>
<div class='form-group' id='kubespawner-profiles-list'>
<label for="args">Notebook envs</label>
<select name="profile">
<option value="1C2G">1C2G</option>
<option value="4C8G">4C8G</option>
<option value="1C2G1X">1C2G1X</option>
</select>
</div>
<div class="form-group">
<label for="args">Extra notebook CLI arguments</label>
<input name="args" class="form-control"
placeholder="e.g. --debug"></input>
</div>
<div class="form-group">
<label for="env">Environment variables</label>
<textarea class="form-control" name="env" placeholder="key=value,key=value"></textarea>
</div>
<div class="form-group">
<label for="sc">Spark Config (one per line)</label>
<textarea class="form-control" name="sc" placeholder="eg:&#10;spark.driver.maxResultSize=4G&#10;spark.kryoserializer.buffer.max=512m&#10;spark.rpc.message.maxSize=2000"></textarea>
</div>
"""
# Register Dask Gateway service and setup singleuser environment.
01-add-dask-gateway-values: |
c.JupyterHub.services = [
{
"name": "admin",
"admin": True,
"api_token": "8890f6d74f63a7b6b35094c9c2e99c1e6db263b56eceb24355238f9f775458f7",
},
{
"name": "dask-gateway",
"api_token": "65be3d7cc1302d24059a9bdd93f03850e8f920ae3475ebab24a5f836b7bacfa3",
},
{
"name": "admin",
"admin": True,
"api_token": "8890f6d74f63a7b6b35094c9c2e99c1e6db263b56eceb24355238f9f775458f7",
},
{
"name": "dask-gateway",
"api_token": "65be3d7cc1302d24059a9bdd93f03850e8f920ae3475ebab24a5f836b7bacfa3",
},
]
# 1. Sets `DASK_GATEWAY__PROXY_ADDRESS` in the singleuser environment.
Expand All @@ -75,122 +118,99 @@ hub:
release_name = os.environ["HELM_RELEASE_NAME"]
release_namespace = os.environ["POD_NAMESPACE"]
public_address = "{}/jupyter/services/dask-gateway/".format(
os.environ["MAGNOTE_HOST"],
)
if "PROXY_HTTP_SERVICE_HOST" in os.environ:
# https is enabled, we want to use the internal http service.
gateway_address = "http://{}:{}/jupyter/services/dask-gateway/".format(
os.environ["PROXY_HTTP_SERVICE_HOST"],
os.environ["PROXY_HTTP_SERVICE_PORT"],
)
print("Setting DASK_GATEWAY__ADDRESS {} from HTTP service".format(gateway_address))
# https is enabled, we want to use the internal http service.
gateway_address = "http://{}:{}/jupyter/services/dask-gateway/".format(
os.environ["PROXY_HTTP_SERVICE_HOST"],
os.environ["PROXY_HTTP_SERVICE_PORT"],
)
print("Setting DASK_GATEWAY__ADDRESS {} from HTTP service".format(gateway_address))
else:
gateway_address = "http://proxy-public/jupyter/services/dask-gateway"
print("Setting DASK_GATEWAY__ADDRESS {}".format(gateway_address))
gateway_address = "http://proxy-public/jupyter/services/dask-gateway"
print("Setting DASK_GATEWAY__ADDRESS {}".format(gateway_address))
# Internal address to connect to the Dask Gateway.
c.KubeSpawner.environment.setdefault("DASK_GATEWAY__ADDRESS", gateway_address)
# Internal address for the Dask Gateway proxy.
c.KubeSpawner.environment.setdefault("DASK_GATEWAY__PROXY_ADDRESS", "gateway://traefik-dask-gateway.dask:80")
# Relative address for the dashboard link.
c.KubeSpawner.environment.setdefault("DASK_GATEWAY__PUBLIC_ADDRESS", "/jupyter/services/dask-gateway/")
c.KubeSpawner.environment.setdefault("DASK_GATEWAY__PUBLIC_ADDRESS", public_address")
# Use JupyterHub to authenticate with Dask Gateway.
c.KubeSpawner.environment.setdefault("DASK_GATEWAY__AUTH__TYPE", "jupyterhub")
# Set dask-labextension to use GatewayCluster
c.KubeSpawner.environment.setdefault("DASK_DISTRIBUTED__DASHBOARD_LINK", "/jupyter/user/{JUPYTERHUB_USER}/proxy/{port}/status")
c.KubeSpawner.environment.setdefault("DASK_DISTRIBUTED__DASHBOARD_LINK", "/user/{JUPYTERHUB_USER}/proxy/{port}/status")
c.KubeSpawner.environment.setdefault("DASK_LABEXTENSION__FACTORY__CLASS", "GatewayCluster")
c.KubeSpawner.environment.setdefault("DASK_LABEXTENSION__FACTORY__MODULE", "dask_gateway")
# Adds Dask Gateway as a JupyterHub service to make the gateway available at
# {HUB_URL}/services/dask-gateway
service_url = "http://traefik-dask-gateway.dask"
for service in c.JupyterHub.services:
if service["name"] == "dask-gateway":
if not service.get("url", None):
print("Adding dask-gateway service URL")
service.setdefault("url", service_url)
break
if service["name"] == "dask-gateway":
if not service.get("url", None):
print("Adding dask-gateway service URL")
service.setdefault("url", service_url)
break
else:
print("dask-gateway service not found. Did you set jupyterhub.hub.services.dask-gateway.apiToken?")
print("dask-gateway service not found. Did you set jupyterhub.hub.services.dask-gateway.apiToken?")
default_env = "1C2G"
c.KubeSpawner.options_form = """
<style>
/* The profile description should not be bold, even though it is inside the <label> tag */
#kubespawner-profiles-list label p {
font-weight: normal;
}
textarea.form-control {
height: 100px;
}
</style>
<div class='form-group' id='kubespawner-profiles-list'>
<label for="args">Notebook envs</label>
<select name="profile">
<option value="1C2G">1C2G</option>
<option value="4C8G">4C8G</option>
<option value="1C2G1X">1C2G1X</option>
</select>
</div>
<div class="form-group">
<label for="args">Extra notebook CLI arguments</label>
<input name="args" class="form-control"
placeholder="e.g. --debug"></input>
</div>
<div class="form-group">
<label for="env">Environment variables</label>
<textarea class="form-control" name="env" placeholder="key=value,key=value"></textarea>
</div>
<div class="form-group">
<label for="sc">Spark Config (one per line)</label>
<textarea class="form-control" name="sc" placeholder="eg:&#10;spark.driver.maxResultSize=4G&#10;spark.kryoserializer.buffer.max=512m&#10;spark.rpc.message.maxSize=2000"></textarea>
</div>
"""
01-add-spwawn-hook: |
02-add-spwawn-hook: |
from kubespawner import KubeSpawner
class FormSpawner(KubeSpawner):
def options_from_form(self, formdata):
options = {}
options['env'] = env = {}
env_lines = formdata.get('env', [''])
print("Setting ENV {}".format(env_lines))
for line in env_lines[0].split(','):
if line:
key, value = line.split('=', 1)
env[key.strip()] = value.strip()
sc_lines = formdata.get('sc', [''])
print("Setting SC {}".format(sc_lines))
for line in sc_lines[0].splitlines():
if line:
key, value = line.split('=', 1)
env['SPARK_CONF__' + key.strip().replace('.', '_')] = value.strip()
arg_s = formdata.get('args', [''])[0].strip()
if arg_s:
options['argv'] = shlex.split(arg_s)
options['profile'] = formdata.get('profile', [None])[0]
return options
def get_args(self):
"""Return arguments to pass to the notebook server"""
argv = super().get_args()
if self.user_options.get('argv'):
argv.extend(self.user_options['argv'])
return argv
def get_env(self):
env = super().get_env()
if self.user_options.get('env'):
env.update(self.user_options['env'])
return env
def options_from_form(self, formdata):
options = {}
options['env'] = env = {}
env_lines = formdata.get('env', [''])
print("Setting ENV {}".format(env_lines))
for line in env_lines[0].split(','):
if line:
key, value = line.split('=', 1)
env[key.strip()] = value.strip()
sc_lines = formdata.get('sc', [''])
print("Setting SC {}".format(sc_lines))
for line in sc_lines[0].splitlines():
if line:
key, value = line.split('=', 1)
env['SPARK_CONF__' + key.strip().replace('.', '_')] = value.strip()
arg_s = formdata.get('args', [''])[0].strip()
if arg_s:
options['argv'] = shlex.split(arg_s)
options['profile'] = formdata.get('profile', [None])[0]
return options
def get_args(self):
"""Return arguments to pass to the notebook server"""
argv = super().get_args()
if self.user_options.get('argv'):
argv.extend(self.user_options['argv'])
return argv
def get_env(self):
env = super().get_env()
if self.user_options.get('env'):
env.update(self.user_options['env'])
return env
c.JupyterHub.spawner_class = FormSpawner
c.KubeSpawner.lifecycle_hooks = {
"postStart": {
"exec": {
"command": ["/bin/sh", "-c", "/usr/local/bin/before-notebook.d/sparkmagic.sh"]
}
}
}
ingress:
enabled: true
hosts:
Expand Down

0 comments on commit f39a050

Please sign in to comment.