diff --git a/gnu/packages/patches/jupyter-unix-domain-sockets-4835-5.7.4.patch b/gnu/packages/patches/jupyter-unix-domain-sockets-4835-5.7.4.patch new file mode 100644 index 0000000000..134d3ad2b8 --- /dev/null +++ b/gnu/packages/patches/jupyter-unix-domain-sockets-4835-5.7.4.patch @@ -0,0 +1,591 @@ +diff -Naur notebook-5.7.4/notebook/base/handlers.py notebook-5.7.4.patched/notebook/base/handlers.py +--- notebook-5.7.4/notebook/base/handlers.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/base/handlers.py 2019-11-18 12:16:58.315065024 +0100 +@@ -40,7 +40,7 @@ + import notebook + from notebook._tz import utcnow + from notebook.i18n import combine_translations +-from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape ++from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape, urldecode_unix_socket_path + from notebook.services.security import csp_report_uri + + #----------------------------------------------------------------------------- +@@ -426,13 +426,18 @@ + # ip_address only accepts unicode on Python 2 + host = host.decode('utf8', 'replace') + +- try: +- addr = ipaddress.ip_address(host) +- except ValueError: +- # Not an IP address: check against hostnames +- allow = host in self.settings.get('local_hostnames', ['localhost']) ++ # UNIX socket handling ++ check_host = urldecode_unix_socket_path(host) ++ if check_host.startswith('/') and os.path.exists(check_host): ++ allow = True + else: +- allow = addr.is_loopback ++ try: ++ addr = ipaddress.ip_address(host) ++ except ValueError: ++ # Not an IP address: check against hostnames ++ allow = host in self.settings.get('local_hostnames', ['localhost']) ++ else: ++ allow = addr.is_loopback + + if not allow: + self.log.warning( +diff -Naur notebook-5.7.4/notebook/__init__.py notebook-5.7.4.patched/notebook/__init__.py +--- notebook-5.7.4/notebook/__init__.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/__init__.py 2019-11-18 12:16:58.315065024 +0100 +@@ -20,6 +20,8 @@ + os.path.join(os.path.dirname(__file__), "templates"), + ] + ++DEFAULT_NOTEBOOK_PORT = 8888 ++ + del os + + from .nbextensions import install_nbextension +diff -Naur notebook-5.7.4/notebook/notebookapp.py notebook-5.7.4.patched/notebook/notebookapp.py +--- notebook-5.7.4/notebook/notebookapp.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/notebookapp.py 2019-11-18 12:21:34.975072928 +0100 +@@ -63,8 +63,11 @@ + from tornado import web + from tornado.httputil import url_concat + from tornado.log import LogFormatter, app_log, access_log, gen_log ++if not sys.platform.startswith('win'): ++ from tornado.netutil import bind_unix_socket + + from notebook import ( ++ DEFAULT_NOTEBOOK_PORT, + DEFAULT_STATIC_FILES_PATH, + DEFAULT_TEMPLATE_PATH_LIST, + __version__, +@@ -108,7 +111,16 @@ + from notebook._sysinfo import get_sys_info + + from ._tz import utcnow, utcfromtimestamp +-from .utils import url_path_join, check_pid, url_escape, urljoin, pathname2url ++from .utils import ( ++ check_pid, ++ pathname2url, ++ url_escape, ++ url_path_join, ++ urldecode_unix_socket_path, ++ urlencode_unix_socket, ++ urlencode_unix_socket_path, ++ urljoin, ++) + + #----------------------------------------------------------------------------- + # Module globals +@@ -212,7 +224,7 @@ + warnings.warn(_("The `ignore_minified_js` flag is deprecated and will be removed in Notebook 6.0"), DeprecationWarning) + + now = utcnow() +- ++ + root_dir = contents_manager.root_dir + home = os.path.expanduser('~') + if root_dir.startswith(home + os.path.sep): +@@ -385,6 +397,7 @@ + set_password(config_file=self.config_file) + self.log.info("Wrote hashed password to %s" % self.config_file) + ++ + def shutdown_server(server_info, timeout=5, log=None): + """Shutdown a notebook server in a separate process. + +@@ -397,14 +410,39 @@ + Returns True if the server was stopped by any means, False if stopping it + failed (on Windows). + """ +- from tornado.httpclient import HTTPClient, HTTPRequest ++ from tornado import gen ++ from tornado.httpclient import AsyncHTTPClient, HTTPClient, HTTPRequest ++ from tornado.netutil import bind_unix_socket, Resolver + url = server_info['url'] + pid = server_info['pid'] ++ resolver = None ++ ++ # UNIX Socket handling. ++ if url.startswith('http+unix://'): ++ # This library doesn't understand our URI form, but it's just HTTP. ++ url = url.replace('http+unix://', 'http://') ++ ++ class UnixSocketResolver(Resolver): ++ def initialize(self, resolver): ++ self.resolver = resolver ++ ++ def close(self): ++ self.resolver.close() ++ ++ @gen.coroutine ++ def resolve(self, host, port, *args, **kwargs): ++ raise gen.Return([ ++ (socket.AF_UNIX, urldecode_unix_socket_path(host)) ++ ]) ++ ++ resolver = UnixSocketResolver(resolver=Resolver()) ++ + req = HTTPRequest(url + 'api/shutdown', method='POST', body=b'', headers={ + 'Authorization': 'token ' + server_info['token'] + }) + if log: log.debug("POST request to %sapi/shutdown", url) +- HTTPClient().fetch(req) ++ AsyncHTTPClient.configure(None, resolver=resolver) ++ HTTPClient(AsyncHTTPClient).fetch(req) + + # Poll to see if it shut down. + for _ in range(timeout*10): +@@ -435,13 +473,20 @@ + version = __version__ + description="Stop currently running notebook server for a given port" + +- port = Integer(8888, config=True, +- help="Port of the server to be killed. Default 8888") ++ port = Integer(DEFAULT_NOTEBOOK_PORT, config=True, ++ help="Port of the server to be killed. Default %s" % DEFAULT_NOTEBOOK_PORT) ++ ++ sock = Unicode(u'', config=True, ++ help="UNIX socket of the server to be killed.") + + def parse_command_line(self, argv=None): + super(NbserverStopApp, self).parse_command_line(argv) + if self.extra_args: +- self.port=int(self.extra_args[0]) ++ try: ++ self.port = int(self.extra_args[0]) ++ except ValueError: ++ # self.extra_args[0] was not an int, so it must be a string (unix socket). ++ self.sock = self.extra_args[0] + + def shutdown_server(self, server): + return shutdown_server(server, log=self.log) +@@ -451,16 +496,16 @@ + if not servers: + self.exit("There are no running servers") + for server in servers: +- if server['port'] == self.port: +- print("Shutting down server on port", self.port, "...") ++ if server.get('sock') == self.sock or server['port'] == self.port: ++ print("Shutting down server on %s..." % self.sock or self.port) + if not self.shutdown_server(server): + sys.exit("Could not stop server") + return + else: + print("There is currently no server running on port {}".format(self.port), file=sys.stderr) +- print("Ports currently in use:", file=sys.stderr) ++ print("Ports/sockets currently in use:", file=sys.stderr) + for server in servers: +- print(" - {}".format(server['port']), file=sys.stderr) ++ print(" - {}".format(server.get('sock', server['port'])), file=sys.stderr) + self.exit(1) + + +@@ -540,6 +585,8 @@ + 'ip': 'NotebookApp.ip', + 'port': 'NotebookApp.port', + 'port-retries': 'NotebookApp.port_retries', ++ 'sock': 'NotebookApp.sock', ++ 'sock-umask': 'NotebookApp.sock_umask', + 'transport': 'KernelManager.transport', + 'keyfile': 'NotebookApp.keyfile', + 'certfile': 'NotebookApp.certfile', +@@ -678,10 +725,18 @@ + or containerized setups for example).""") + ) + +- port = Integer(8888, config=True, ++ port = Integer(DEFAULT_NOTEBOOK_PORT, config=True, + help=_("The port the notebook server will listen on.") + ) + ++ sock = Unicode(u'', config=True, ++ help=_("The UNIX socket the notebook server will listen on.") ++ ) ++ ++ sock_umask = Unicode(u'0600', config=True, ++ help=_("The UNIX socket umask to set on creation (default: 0600).") ++ ) ++ + port_retries = Integer(50, config=True, + help=_("The number of additional ports to try if the specified port is not available.") + ) +@@ -1370,6 +1425,27 @@ + self.log.critical(_("\t$ python -m notebook.auth password")) + sys.exit(1) + ++ # Socket options validation. ++ if self.sock: ++ if self.port != DEFAULT_NOTEBOOK_PORT: ++ self.log.critical( ++ _('Options --port and --sock are mutually exclusive. Aborting.'), ++ ) ++ sys.exit(1) ++ ++ if self.open_browser: ++ # If we're bound to a UNIX socket, we can't reliably connect from a browser. ++ self.log.critical( ++ _('Options --open-browser and --sock are mutually exclusive. Aborting.'), ++ ) ++ sys.exit(1) ++ ++ if sys.platform.startswith('win'): ++ self.log.critical( ++ _('Option --sock is not supported on Windows, but got value of %s. Aborting.' % self.sock), ++ ) ++ sys.exit(1) ++ + self.web_app = NotebookWebApplication( + self, self.kernel_manager, self.contents_manager, + self.session_manager, self.kernel_spec_manager, +@@ -1401,6 +1477,32 @@ + max_body_size=self.max_body_size, + max_buffer_size=self.max_buffer_size) + ++ success = self._bind_http_server() ++ if not success: ++ self.log.critical(_('ERROR: the notebook server could not be started because ' ++ 'no available port could be found.')) ++ self.exit(1) ++ ++ def _bind_http_server(self): ++ return self._bind_http_server_unix() if self.sock else self._bind_http_server_tcp() ++ ++ def _bind_http_server_unix(self): ++ try: ++ sock = bind_unix_socket(self.sock, mode=int(self.sock_umask.encode(), 8)) ++ self.http_server.add_socket(sock) ++ except socket.error as e: ++ if e.errno == errno.EADDRINUSE: ++ self.log.info(_('The socket %s is already in use.') % self.sock) ++ return False ++ elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): ++ self.log.warning(_("Permission to listen on sock %s denied") % self.sock) ++ return False ++ else: ++ raise ++ else: ++ return True ++ ++ def _bind_http_server_tcp(self): + success = None + for port in random_ports(self.port, self.port_retries+1): + try: +@@ -1418,10 +1520,11 @@ + self.port = port + success = True + break +- if not success: +- self.log.critical(_('ERROR: the notebook server could not be started because ' +- 'no available port could be found.')) +- self.exit(1) ++ return success ++ ++ def _concat_token(self, url): ++ token = self.token if self._token_generated else '...' ++ return url_concat(url, {'token': token}) + + @property + def display_url(self): +@@ -1429,26 +1532,33 @@ + url = self.custom_display_url + if not url.endswith('/'): + url += '/' ++ elif self.sock: ++ url = self._unix_sock_url() + else: + if self.ip in ('', '0.0.0.0'): + ip = "(%s or 127.0.0.1)" % socket.gethostname() + else: + ip = self.ip +- url = self._url(ip) +- if self.token: +- # Don't log full token if it came from config +- token = self.token if self._token_generated else '...' +- url = url_concat(url, {'token': token}) ++ url = self._tcp_url(ip) ++ if self.token and not self.sock: ++ url = self._concat_token(url) ++ url += '\n or %s' % self._concat_token(self._tcp_url('127.0.0.1')) + return url + + @property + def connection_url(self): +- ip = self.ip if self.ip else 'localhost' +- return self._url(ip) ++ if self.sock: ++ return self._unix_sock_url() ++ else: ++ ip = self.ip if self.ip else 'localhost' ++ return self._tcp_url(ip) + +- def _url(self, ip): ++ def _unix_sock_url(self, token=None): ++ return '%s%s' % (urlencode_unix_socket(self.sock), self.base_url) ++ ++ def _tcp_url(self, ip, port=None): + proto = 'https' if self.certfile else 'http' +- return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) ++ return "%s://%s:%i%s" % (proto, ip, port or self.port, self.base_url) + + def init_terminals(self): + if not self.terminals_enabled: +@@ -1660,6 +1770,7 @@ + return {'url': self.connection_url, + 'hostname': self.ip if self.ip else 'localhost', + 'port': self.port, ++ 'sock': self.sock, + 'secure': bool(self.certfile), + 'base_url': self.base_url, + 'token': self.token, +@@ -1780,19 +1891,31 @@ + self.write_server_info_file() + self.write_browser_open_file() + +- if self.open_browser or self.file_to_run: ++ if (self.open_browser or self.file_to_run) and not self.sock: + self.launch_browser() + + if self.token and self._token_generated: + # log full URL with generated token, so there's a copy/pasteable link + # with auth info. +- self.log.critical('\n'.join([ +- '\n', +- 'To access the notebook, open this file in a browser:', +- ' %s' % urljoin('file:', pathname2url(self.browser_open_file)), +- 'Or copy and paste one of these URLs:', +- ' %s' % self.display_url, +- ])) ++ if self.sock: ++ self.log.critical('\n'.join([ ++ '\n', ++ 'Notebook is listening on %s' % self.display_url, ++ '', ++ ( ++ 'UNIX sockets are not browser-connectable, but you can tunnel to ' ++ 'the instance via e.g.`ssh -L 8888:%s -N user@this_host` and then ' ++ 'opening e.g. %s in a browser.' ++ ) % (self.sock, self._concat_token(self._tcp_url('localhost', 8888))) ++ ])) ++ else: ++ self.log.critical('\n'.join([ ++ '\n', ++ 'To access the notebook, open this file in a browser:', ++ ' %s' % urljoin('file:', pathname2url(self.browser_open_file)), ++ 'Or copy and paste one of these URLs:', ++ ' %s' % self.display_url, ++ ])) + + self.io_loop = ioloop.IOLoop.current() + if sys.platform.startswith('win'): +diff -Naur notebook-5.7.4/notebook/tests/launchnotebook.py notebook-5.7.4.patched/notebook/tests/launchnotebook.py +--- notebook-5.7.4/notebook/tests/launchnotebook.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/tests/launchnotebook.py 2019-11-18 12:22:25.931074384 +0100 +@@ -19,12 +19,13 @@ + from mock import patch #py2 + + import requests ++import requests_unixsocket + from tornado.ioloop import IOLoop + import zmq + + import jupyter_core.paths + from traitlets.config import Config +-from ..notebookapp import NotebookApp ++from ..notebookapp import NotebookApp, urlencode_unix_socket + from ..utils import url_path_join + from ipython_genutils.tempdir import TemporaryDirectory + +@@ -55,7 +56,7 @@ + url = cls.base_url() + 'api/contents' + for _ in range(int(MAX_WAITTIME/POLL_INTERVAL)): + try: +- requests.get(url) ++ cls.fetch_url(url) + except Exception as e: + if not cls.notebook_thread.is_alive(): + raise RuntimeError("The notebook server failed to start") +@@ -79,6 +80,10 @@ + headers['Authorization'] = 'token %s' % cls.token + return headers + ++ @staticmethod ++ def fetch_url(url): ++ return requests.get(url) ++ + @classmethod + def request(cls, verb, path, **kwargs): + """Send a request to my server +@@ -93,6 +98,10 @@ + return response + + @classmethod ++ def get_bind_args(cls): ++ return dict(port=cls.port) ++ ++ @classmethod + def setup_class(cls): + cls.tmp_dir = TemporaryDirectory() + def tmp(*parts): +@@ -103,7 +112,7 @@ + if e.errno != errno.EEXIST: + raise + return path +- ++ + cls.home_dir = tmp('home') + data_dir = cls.data_dir = tmp('data') + config_dir = cls.config_dir = tmp('config') +@@ -138,8 +147,8 @@ + if 'asyncio' in sys.modules: + import asyncio + asyncio.set_event_loop(asyncio.new_event_loop()) ++ bind_args = cls.get_bind_args() + app = cls.notebook = NotebookApp( +- port=cls.port, + port_retries=0, + open_browser=False, + config_dir=cls.config_dir, +@@ -150,6 +159,7 @@ + config=config, + allow_root=True, + token=cls.token, ++ **bind_args + ) + # don't register signal handler during tests + app.init_signal = lambda : None +@@ -197,6 +207,25 @@ + return 'http://localhost:%i%s' % (cls.port, cls.url_prefix) + + ++class UNIXSocketNotebookTestBase(NotebookTestBase): ++ # Rely on `/tmp` to avoid any Linux socket length max buffer ++ # issues. Key on PID for process-wise concurrency. ++ sock = '/tmp/.notebook.%i.sock' % os.getpid() ++ ++ @classmethod ++ def get_bind_args(cls): ++ return dict(sock=cls.sock) ++ ++ @classmethod ++ def base_url(cls): ++ return '%s%s' % (urlencode_unix_socket(cls.sock), cls.url_prefix) ++ ++ @staticmethod ++ def fetch_url(url): ++ with requests_unixsocket.monkeypatch(): ++ return requests.get(url) ++ ++ + @contextmanager + def assert_http_error(status, msg=None): + try: +diff -Naur notebook-5.7.4/notebook/tests/test_notebookapp_integration.py notebook-5.7.4.patched/notebook/tests/test_notebookapp_integration.py +--- notebook-5.7.4/notebook/tests/test_notebookapp_integration.py 1970-01-01 01:00:00.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/tests/test_notebookapp_integration.py 2019-11-18 12:16:58.319065025 +0100 +@@ -0,0 +1,39 @@ ++import os ++import stat ++import subprocess ++import time ++ ++from ipython_genutils.testing.decorators import skip_win32 ++ ++from .launchnotebook import UNIXSocketNotebookTestBase ++from ..utils import urlencode_unix_socket, urlencode_unix_socket_path ++ ++ ++@skip_win32 ++def test_shutdown_sock_server_integration(): ++ sock = UNIXSocketNotebookTestBase.sock ++ url = urlencode_unix_socket(sock) ++ encoded_sock_path = urlencode_unix_socket_path(sock) ++ ++ p = subprocess.Popen( ++ ['jupyter', 'notebook', '--no-browser', '--sock=%s' % sock], ++ stdout=subprocess.PIPE, stderr=subprocess.PIPE ++ ) ++ ++ for line in iter(p.stderr.readline, b''): ++ if url.encode() in line: ++ complete = True ++ break ++ ++ assert complete, 'did not find socket URL in stdout when launching notebook' ++ ++ assert encoded_sock_path.encode() in subprocess.check_output(['jupyter', 'notebook', 'list']) ++ ++ # Ensure default umask is properly applied. ++ assert stat.S_IMODE(os.lstat(sock).st_mode) == 0o600 ++ ++ subprocess.check_output(['jupyter', 'notebook', 'stop', sock]) ++ ++ assert encoded_sock_path.encode() not in subprocess.check_output(['jupyter', 'notebook', 'list']) ++ ++ p.wait() +diff -Naur notebook-5.7.4/notebook/tests/test_notebookapp.py notebook-5.7.4.patched/notebook/tests/test_notebookapp.py +--- notebook-5.7.4/notebook/tests/test_notebookapp.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/tests/test_notebookapp.py 2019-11-18 12:16:58.319065025 +0100 +@@ -25,7 +25,7 @@ + from notebook.auth.security import passwd_check + NotebookApp = notebookapp.NotebookApp + +-from .launchnotebook import NotebookTestBase ++from .launchnotebook import NotebookTestBase, UNIXSocketNotebookTestBase + + + def test_help_output(): +@@ -192,3 +192,15 @@ + servers = list(notebookapp.list_running_servers()) + assert len(servers) >= 1 + assert self.port in {info['port'] for info in servers} ++ ++ ++# UNIX sockets aren't available on Windows. ++if not sys.platform.startswith('win'): ++ class NotebookUnixSocketTests(UNIXSocketNotebookTestBase): ++ def test_run(self): ++ self.fetch_url(self.base_url() + 'api/contents') ++ ++ def test_list_running_sock_servers(self): ++ servers = list(notebookapp.list_running_servers()) ++ assert len(servers) >= 1 ++ assert self.sock in {info['sock'] for info in servers} +diff -Naur notebook-5.7.4/notebook/utils.py notebook-5.7.4.patched/notebook/utils.py +--- notebook-5.7.4/notebook/utils.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/notebook/utils.py 2019-11-18 12:23:05.231075507 +0100 +@@ -306,3 +306,18 @@ + check_pid = _check_pid_win32 + else: + check_pid = _check_pid_posix ++ ++def urlencode_unix_socket_path(socket_path): ++ """Encodes a UNIX socket path string from a socket path for the `http+unix` URI form.""" ++ return socket_path.replace('/', '%2F') ++ ++ ++def urldecode_unix_socket_path(socket_path): ++ """Decodes a UNIX sock path string from an encoded sock path for the `http+unix` URI form.""" ++ return socket_path.replace('%2F', '/') ++ ++ ++def urlencode_unix_socket(socket_path): ++ """Encodes a UNIX socket URL from a socket path for the `http+unix` URI form.""" ++ return 'http+unix://%s' % urlencode_unix_socket_path(socket_path) ++ +diff -Naur notebook-5.7.4/setup.py notebook-5.7.4.patched/setup.py +--- notebook-5.7.4/setup.py 2018-12-17 11:01:51.000000000 +0100 ++++ notebook-5.7.4.patched/setup.py 2019-11-18 12:23:33.851076325 +0100 +@@ -98,7 +98,8 @@ + ':python_version == "2.7"': ['ipaddress'], + 'test:python_version == "2.7"': ['mock'], + 'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', +- 'nbval', 'nose-exclude', 'selenium'], ++ 'nbval', 'nose-exclude', 'selenium', ++ 'requests-unixsocket'], + 'test:sys_platform == "win32"': ['nose-exclude'], + }, + entry_points = { diff --git a/gnu/packages/python-xyz.scm b/gnu/packages/python-xyz.scm index 885d2f3bb5..fed744ac77 100644 --- a/gnu/packages/python-xyz.scm +++ b/gnu/packages/python-xyz.scm @@ -7811,7 +7811,8 @@ convert an @code{.ipynb} notebook file into various static formats including: (uri (pypi-uri "notebook" version)) (sha256 (base32 - "0jm7324mbxljmn9hgapj66q7swyz5ai92blmr0jpcy0h80x6f26r")))) + "0jm7324mbxljmn9hgapj66q7swyz5ai92blmr0jpcy0h80x6f26r")) + (patches (search-patches "jupyter-unix-domain-sockets-4835-5.7.4.patch")))) (build-system python-build-system) (arguments `(#:phases