From 648cf4c76fae1460fdbcfd041779ce9b7ba2a538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 28 Mar 2019 12:06:25 -0400 Subject: [PATCH 1/7] Replace current_ip and current_port by ip and port --- batchspawner/api.py | 9 +++++---- batchspawner/batchspawner.py | 23 ++++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/batchspawner/api.py b/batchspawner/api.py index f78693a4..109e3ba7 100644 --- a/batchspawner/api.py +++ b/batchspawner/api.py @@ -5,12 +5,13 @@ class BatchSpawnerAPIHandler(APIHandler): @web.authenticated def post(self): - """POST set user's spawner port number""" + """POST set user spawner data""" user = self.get_current_user() data = self.get_json_body() - port = int(data.get('port', 0)) - user.spawner.current_port = port - self.finish(json.dumps({"message": "BatchSpawner port configured"})) + for key, value in data.items(): + if hasattr(user.spawner, key): + setattr(user.spawner, key, value) + self.finish(json.dumps({"message": "BatchSpawner data configured"})) self.set_status(201) default_handlers.append((r"/api/batchspawner", BatchSpawnerAPIHandler)) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 1a8a77b5..885e020f 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -165,12 +165,6 @@ def _req_keepvars_default(self): # Will get the raw output of the job status command unless overridden job_status = Unicode() - # Will get the address of the server as reported by job manager - current_ip = Unicode() - - # Will get the port of the server as reported by singleserver - current_port = Integer() - # Prepare substitution variables for templates using req_xyz traits def get_req_subvars(self): reqlist = [ t for t in self.trait_names() if t.startswith('req_') ] @@ -349,6 +343,9 @@ def poll(self): @gen.coroutine def start(self): """Start the process""" + self.ip = self.traits()['ip'].default_value + self.port = self.traits()['port'].default_value + if jupyterhub.version_info >= (0,8) and self.server: self.server.port = self.port @@ -375,20 +372,20 @@ def start(self): ' after starting.') yield gen.sleep(self.startup_poll_interval) - self.current_ip = self.state_gethost() - while self.current_port == 0: + self.ip = self.state_gethost() + while self.port == 0: yield gen.sleep(self.startup_poll_interval) if jupyterhub.version_info < (0,7): # store on user for pre-jupyterhub-0.7: - self.user.server.port = self.current_port - self.user.server.ip = self.current_ip + self.user.server.port = self.port + self.user.server.ip = self.ip self.db.commit() self.log.info("Notebook server job {0} started at {1}:{2}".format( - self.job_id, self.current_ip, self.current_port) + self.job_id, self.ip, self.port) ) - return self.current_ip, self.current_port + return self.ip, self.port @gen.coroutine def stop(self, now=False): @@ -408,7 +405,7 @@ def stop(self, now=False): yield gen.sleep(1.0) if self.job_id: self.log.warn("Notebook server job {0} at {1}:{2} possibly failed to terminate".format( - self.job_id, self.current_ip, self.port) + self.job_id, self.ip, self.port) ) From a5e55bb5eea1e1d2153075d700ba0689268d47b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 24 Apr 2019 16:28:56 -0400 Subject: [PATCH 2/7] Allow batchspawner to be used with different notebook app Instead of creating a batchspawner notebook app, batchspawner-singleuser is now a wrapper that finds a port and add the port number to the command-line argument of the singleuser app. This allows the usage of batchspawner with jupyterlab for example. --- batchspawner/batchspawner.py | 5 +---- batchspawner/singleuser.py | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 1a8a77b5..9e320f8f 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -74,9 +74,6 @@ class BatchSpawnerBase(Spawner): state_gethost """ - # override default since will need to set the listening port using the api - cmd = Command(['batchspawner-singleuser'], allow_none=True).tag(config=True) - # override default since batch systems typically need longer start_timeout = Integer(300).tag(config=True) @@ -190,7 +187,7 @@ def parse_job_id(self, output): return output def cmd_formatted_for_batch(self): - return ' '.join(self.cmd + self.get_args()) + return ' '.join(['batchspawner-singleuser'] + self.cmd + self.get_args()) @gen.coroutine def run_command(self, cmd, input=None, env=None): diff --git a/batchspawner/singleuser.py b/batchspawner/singleuser.py index 6d493262..2d76f2eb 100644 --- a/batchspawner/singleuser.py +++ b/batchspawner/singleuser.py @@ -1,21 +1,21 @@ -from jupyterhub.singleuser import SingleUserNotebookApp -from jupyterhub.utils import random_port, url_path_join -from traitlets import default +import sys -class BatchSingleUserNotebookApp(SingleUserNotebookApp): - @default('port') - def _port(self): - return random_port() +from runpy import run_path +from shutil import which - def start(self): - # Send Notebook app's port number to remote Spawner - self.hub_auth._api_request(method='POST', - url=url_path_join(self.hub_api_url, 'batchspawner'), - json={'port' : self.port}) - super().start() +from jupyterhub.utils import random_port, url_path_join +from jupyterhub.services.auth import HubAuth def main(argv=None): - return BatchSingleUserNotebookApp.launch_instance(argv) + port = random_port() + hub_auth = HubAuth() + hub_auth._api_request(method='POST', + url=url_path_join(hub_auth.api_url, 'batchspawner'), + json={'port' : port}) + + cmd_path = which(sys.argv[1]) + sys.argv = sys.argv[1:] + ['--port={}'.format(port)] + run_path(cmd_path, run_name="__main__") if __name__ == "__main__": main() \ No newline at end of file From e6d4bfa5e772a189ef5ec9b9540d154806fa824b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Mon, 27 May 2019 13:58:19 -0400 Subject: [PATCH 3/7] Add read of SSL environment variable for HubAuth --- batchspawner/singleuser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/batchspawner/singleuser.py b/batchspawner/singleuser.py index 2d76f2eb..afb171a2 100644 --- a/batchspawner/singleuser.py +++ b/batchspawner/singleuser.py @@ -1,3 +1,4 @@ +import os import sys from runpy import run_path @@ -9,6 +10,9 @@ def main(argv=None): port = random_port() hub_auth = HubAuth() + hub_auth.client_ca = os.environ.get('JUPYTERHUB_SSL_CLIENT_CA', '') + hub_auth.certfile = os.environ.get('JUPYTERHUB_SSL_CERTFILE', '') + hub_auth.keyfile = os.environ.get('JUPYTERHUB_SSL_KEYFILE', '') hub_auth._api_request(method='POST', url=url_path_join(hub_auth.api_url, 'batchspawner'), json={'port' : port}) From c80ed1fc6e59d676d3b939846322ca54e8f729c4 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Sun, 3 Feb 2019 00:51:42 +0200 Subject: [PATCH 4/7] Bugfix: Jupyterhub master (beyond 0.9.4) makes get_current_user coroutine --- batchspawner/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/batchspawner/api.py b/batchspawner/api.py index 109e3ba7..bbaacbcd 100644 --- a/batchspawner/api.py +++ b/batchspawner/api.py @@ -6,7 +6,12 @@ class BatchSpawnerAPIHandler(APIHandler): @web.authenticated def post(self): """POST set user spawner data""" - user = self.get_current_user() + if hasattr(self, 'current_user'): + # Jupyterhub compatability, (september 2018, d79a99323ef1d) + user = self.current_user + else: + # Previous jupyterhub, 0.9.4 and before. + user = self.get_current_user() data = self.get_json_body() for key, value in data.items(): if hasattr(user.spawner, key): From f0b091f27af7ea8b84537771bdb992b0125a1c9b Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Fri, 14 Jun 2019 11:28:42 +0300 Subject: [PATCH 5/7] Set port during tests - #139 changes port handling, this makes tests pass. - Sets attribute mock_port, which gets used during the port polling phase. --- batchspawner/batchspawner.py | 4 ++++ batchspawner/tests/test_spawners.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 885e020f..ed9d01b1 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -375,6 +375,10 @@ def start(self): self.ip = self.state_gethost() while self.port == 0: yield gen.sleep(self.startup_poll_interval) + # Test framework: For testing, mock_port is set because we + # don't actually run the single-user server yet. + if hasattr(self, 'mock_port'): + self.port = self.mock_port if jupyterhub.version_info < (0,7): # store on user for pre-jupyterhub-0.7: diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index 672ce395..2941e05d 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -64,7 +64,7 @@ def new_spawner(db, spawner_class=BatchDummy, **kwargs): kwargs.setdefault('server', server) kwargs.setdefault('hub', hub) kwargs.setdefault('user', user) - kwargs.setdefault('current_port', testport) + kwargs.setdefault('mock_port', testport) kwargs.setdefault('INTERRUPT_TIMEOUT', 1) kwargs.setdefault('TERM_TIMEOUT', 1) kwargs.setdefault('KILL_TIMEOUT', 1) From ec4e0bcfa2515ad33249a2f02a3c9bdb0f78a51b Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Fri, 14 Jun 2019 11:31:58 +0300 Subject: [PATCH 6/7] tests: fix ip check after the rename current_ip->ip - Fixes tests for PR #139 --- batchspawner/tests/test_spawners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index 2941e05d..099e52dc 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -84,7 +84,7 @@ def check_ip(spawner, value): if version_info < (0,7): assert spawner.user.server.ip == value else: - assert spawner.current_ip == value + assert spawner.ip == value def test_spawner_start_stop_poll(db, io_loop): spawner = new_spawner(db=db) From ad6b010400dd0fbdfc9d5bfafb51bbdf9dbdb6c6 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Fri, 14 Jun 2019 12:04:35 +0300 Subject: [PATCH 7/7] Update tests for #141: adding batchspawner-singleuser as wrapper - This fixes up tests for #141, which adds batchspawner-singleuser which changes regular expressions that need to be matched in the batch scripts. --- batchspawner/tests/test_spawners.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index 672ce395..2b9285c8 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -287,7 +287,7 @@ def test_torque(db, io_loop): 'req_epilogue': 'EPILOGUE', } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^singleuser_command.*^EPILOGUE', re.S|re.M), + re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), re.compile(r'mem=5678'), re.compile(r'ppn=5'), re.compile(r'^#PBS some_option_asdf', re.M), @@ -315,7 +315,7 @@ def test_moab(db, io_loop): 'req_epilogue': 'EPILOGUE', } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^singleuser_command.*^EPILOGUE', re.S|re.M), + re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), re.compile(r'mem=5678'), re.compile(r'ppn=5'), re.compile(r'^#PBS some_option_asdf', re.M), @@ -376,7 +376,7 @@ def test_slurm(db, io_loop): 'req_reservation': 'RES123', } batch_script_re_list = [ - re.compile(r'PROLOGUE.*srun singleuser_command.*EPILOGUE', re.S), + re.compile(r'PROLOGUE.*srun batchspawner-singleuser singleuser_command.*EPILOGUE', re.S), re.compile(r'^#SBATCH \s+ --cpus-per-task=5', re.X|re.M), re.compile(r'^#SBATCH \s+ --time=3-05:10:10', re.X|re.M), re.compile(r'^#SBATCH \s+ some_option_asdf', re.X|re.M), @@ -441,7 +441,7 @@ def test_condor(db, io_loop): 'req_options': 'some_option_asdf', } batch_script_re_list = [ - re.compile(r'exec singleuser_command'), + re.compile(r'exec batchspawner-singleuser singleuser_command'), re.compile(r'RequestCpus = 5'), re.compile(r'RequestMemory = 5678'), re.compile(r'^some_option_asdf', re.M), @@ -470,7 +470,7 @@ def test_lfs(db, io_loop): 'req_epilogue': 'EPILOGUE', } batch_script_re_list = [ - re.compile(r'^PROLOGUE.*^singleuser_command.*^EPILOGUE', re.S|re.M), + re.compile(r'^PROLOGUE.*^batchspawner-singleuser singleuser_command.*^EPILOGUE', re.S|re.M), re.compile(r'#BSUB\s+-q\s+some_queue', re.M), ] script = [