diff --git a/CHANGES.txt b/CHANGES.txt index c0681053dd..4fdd7bf2f7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,16 @@ unreleased Backward Incompatibilities -------------------------- + - Following the Pyramid deprecation period (1.6 -> 1.8), + daemon support for pserve has been removed. This includes removing the + daemon commands (start, stop, restart, status) as well as the following + arguments: + --daemon --pid-file --log-file --monitor-restart --status --user --group + --stop-daemon + + To run your server as a daemon you should use a process manager instead of + pserve. + Features -------- diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index d40d7d79ca..16023c821b 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -271,4 +271,6 @@ Contributors - Berker Peksag, 2016/05/16 +- Zack Brunson, 2016/06/02 + - Marco Martinez, 2016/06/02 diff --git a/pyramid/scripts/pserve.py b/pyramid/scripts/pserve.py index 74bda1dce4..ec7f317049 100644 --- a/pyramid/scripts/pserve.py +++ b/pyramid/scripts/pserve.py @@ -10,8 +10,6 @@ import atexit import ctypes -import errno -import logging import optparse import os import py_compile @@ -59,20 +57,13 @@ def main(argv=sys.argv, quiet=False): command = PServeCommand(argv, quiet=quiet) return command.run() -class DaemonizeException(Exception): - pass - class PServeCommand(object): - usage = '%prog config_uri [start|stop|restart|status] [var=value]' + usage = '%prog config_uri [var=value]' description = """\ This command serves a web application that uses a PasteDeploy configuration file for the server and application. - If start/stop/restart is given, then --daemon is implied, and it will - start (normal operation), stop (--stop-daemon), or do both. - Note: Daemonization features are deprecated. - You can also include variable assignments like 'http_port=8080' and then use %(http_port)s in your config files. """ @@ -98,23 +89,6 @@ class PServeCommand(object): metavar='SECTION_NAME', help=("Use the named server as defined in the configuration file " "(default: main)")) - if hasattr(os, 'fork'): - parser.add_option( - '--daemon', - dest="daemon", - action="store_true", - help="Run in daemon (background) mode [DEPRECATED]") - parser.add_option( - '--pid-file', - dest='pid_file', - metavar='FILENAME', - help=("Save PID to file (default to pyramid.pid if running in " - "daemon mode) [DEPRECATED]")) - parser.add_option( - '--log-file', - dest='log_file', - metavar='LOG_FILE', - help="Save output to the given log file (redirects stdout) [DEPRECATED]") parser.add_option( '--reload', dest='reload', @@ -126,22 +100,11 @@ class PServeCommand(object): default=1, help=("Seconds between checking files (low number can cause " "significant CPU usage)")) - parser.add_option( - '--monitor-restart', - dest='monitor_restart', - action='store_true', - help="Auto-restart server if it dies [DEPRECATED]") parser.add_option( '-b', '--browser', dest='browser', action='store_true', help="Open a web browser to server url") - parser.add_option( - '--status', - action='store_true', - dest='show_status', - help=("Show the status of the (presumably daemonized) server " - "[DEPRECATED]")) parser.add_option( '-v', '--verbose', default=default_verbosity, @@ -155,33 +118,11 @@ class PServeCommand(object): dest='verbose', help="Suppress verbose output") - if hasattr(os, 'setuid'): - # I don't think these are available on Windows - parser.add_option( - '--user', - dest='set_user', - metavar="USERNAME", - help="Set the user (usually only possible when run as root)") - parser.add_option( - '--group', - dest='set_group', - metavar="GROUP", - help="Set the group (usually only possible when run as root)") - - parser.add_option( - '--stop-daemon', - dest='stop_daemon', - action='store_true', - help=('Stop a daemonized server (given a PID file, or default ' - 'pyramid.pid file) [DEPRECATED]')) - _scheme_re = re.compile(r'^[a-z][a-z]+:', re.I) _reloader_environ_key = 'PYTHON_RELOADER_SHOULD_RUN' _monitor_environ_key = 'PASTE_MONITOR_SHOULD_RUN' - possible_subcommands = ('start', 'stop', 'restart', 'status') - def __init__(self, argv, quiet=False): self.options, self.args = self.parser.parse_args(argv[1:]) if quiet: @@ -192,51 +133,16 @@ def out(self, msg): # pragma: no cover print(msg) def get_options(self): - if ( - len(self.args) > 1 and - self.args[1] in self.possible_subcommands - ): - restvars = self.args[2:] - else: - restvars = self.args[1:] - + restvars = self.args[1:] return parse_vars(restvars) def run(self): # pragma: no cover - if self.options.stop_daemon: - self._warn_daemon_deprecated() - return self.stop_daemon() - - if not hasattr(self.options, 'set_user'): - # Windows case: - self.options.set_user = self.options.set_group = None - - # @@: Is this the right stage to set the user at? - if self.options.set_user or self.options.set_group: - self.change_user_group( - self.options.set_user, self.options.set_group) - if not self.args: self.out('You must give a config file') return 2 app_spec = self.args[0] - if ( - len(self.args) > 1 and - self.args[1] in self.possible_subcommands - ): - cmd = self.args[1] - else: - cmd = None - if self.options.reload: - if ( - getattr(self.options, 'daemon', False) or - cmd in ('start', 'stop', 'restart') - ): - self.out( - 'Error: Cannot use reloading while running as a dameon.') - return 2 if os.environ.get(self._reloader_environ_key): if self.options.verbose > 1: self.out('Running reloading file monitor') @@ -246,31 +152,6 @@ def run(self): # pragma: no cover else: return self.restart_with_reloader() - if cmd not in (None, 'start', 'stop', 'restart', 'status'): - self.out( - 'Error: must give start|stop|restart (not %s)' % cmd) - return 2 - - if cmd == 'status' or self.options.show_status: - self._warn_daemon_deprecated() - return self.show_status() - - if cmd in ('restart', 'stop'): - self._warn_daemon_deprecated() - result = self.stop_daemon() - if result: - if cmd == 'restart': - self.out("Could not stop daemon; aborting") - else: - self.out("Could not stop daemon") - return result - if cmd == 'stop': - return result - self.options.daemon = True - - if cmd == 'start': - self.options.daemon = True - app_name = self.options.app_name vars = self.get_options() @@ -286,75 +167,6 @@ def run(self): # pragma: no cover server_spec = app_spec base = os.getcwd() - # warn before setting a default - if self.options.pid_file or self.options.log_file: - self._warn_daemon_deprecated() - - if getattr(self.options, 'daemon', False): - if not self.options.pid_file: - self.options.pid_file = 'pyramid.pid' - if not self.options.log_file: - self.options.log_file = 'pyramid.log' - - # Ensure the log file is writeable - if self.options.log_file: - try: - writeable_log_file = open(self.options.log_file, 'a') - except IOError as ioe: - msg = 'Error: Unable to write to log file: %s' % ioe - raise ValueError(msg) - writeable_log_file.close() - - # Ensure the pid file is writeable - if self.options.pid_file: - try: - writeable_pid_file = open(self.options.pid_file, 'a') - except IOError as ioe: - msg = 'Error: Unable to write to pid file: %s' % ioe - raise ValueError(msg) - writeable_pid_file.close() - - # warn before forking - if ( - self.options.monitor_restart and - not os.environ.get(self._monitor_environ_key) - ): - self.out('''\ ---monitor-restart has been deprecated in Pyramid 1.6. It will be removed -in a future release per Pyramid's deprecation policy. Please consider using -a real process manager for your processes like Systemd, Circus, or Supervisor. -''') - - if ( - getattr(self.options, 'daemon', False) and - not os.environ.get(self._monitor_environ_key) - ): - self._warn_daemon_deprecated() - try: - self.daemonize() - except DaemonizeException as ex: - if self.options.verbose > 0: - self.out(str(ex)) - return 2 - - if ( - not os.environ.get(self._monitor_environ_key) and - self.options.pid_file - ): - self.record_pid(self.options.pid_file) - - if ( - self.options.monitor_restart and - not os.environ.get(self._monitor_environ_key) - ): - return self.restart_with_monitor() - - if self.options.log_file: - stdout_log = LazyWriter(self.options.log_file, 'a') - sys.stdout = stdout_log - sys.stderr = stdout_log - logging.basicConfig(stream=stdout_log) - log_fn = app_spec if log_fn.startswith('config:'): log_fn = app_spec[len('config:'):] @@ -442,141 +254,6 @@ def find_script_path(self, name): # pragma: no cover name += '.exe' return name - def daemonize(self): # pragma: no cover - pid = live_pidfile(self.options.pid_file) - if pid: - raise DaemonizeException( - "Daemon is already running (PID: %s from PID file %s)" - % (pid, self.options.pid_file)) - - if self.options.verbose > 0: - self.out('Entering daemon mode') - pid = os.fork() - if pid: - # The forked process also has a handle on resources, so we - # *don't* want proper termination of the process, we just - # want to exit quick (which os._exit() does) - os._exit(0) - # Make this the session leader - os.setsid() - # Fork again for good measure! - pid = os.fork() - if pid: - os._exit(0) - - # @@: Should we set the umask and cwd now? - - import resource # Resource usage information. - maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if (maxfd == resource.RLIM_INFINITY): - maxfd = MAXFD - # Iterate through and close all file descriptors. - for fd in range(0, maxfd): - try: - os.close(fd) - except OSError: # ERROR, fd wasn't open to begin with (ignored) - pass - - if (hasattr(os, "devnull")): - REDIRECT_TO = os.devnull - else: - REDIRECT_TO = "/dev/null" - os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) - # Duplicate standard input to standard output and standard error. - os.dup2(0, 1) # standard output (1) - os.dup2(0, 2) # standard error (2) - - def _remove_pid_file(self, written_pid, filename, verbosity): - current_pid = os.getpid() - if written_pid != current_pid: - # A forked process must be exiting, not the process that - # wrote the PID file - return - if not os.path.exists(filename): - return - with open(filename) as f: - content = f.read().strip() - try: - pid_in_file = int(content) - except ValueError: - pass - else: - if pid_in_file != current_pid: - msg = "PID file %s contains %s, not expected PID %s" - self.out(msg % (filename, pid_in_file, current_pid)) - return - if verbosity > 0: - self.out("Removing PID file %s" % filename) - try: - os.unlink(filename) - return - except OSError as e: - # Record, but don't give traceback - self.out("Cannot remove PID file: (%s)" % e) - # well, at least lets not leave the invalid PID around... - try: - with open(filename, 'w') as f: - f.write('') - except OSError as e: - self.out('Stale PID left in file: %s (%s)' % (filename, e)) - else: - self.out('Stale PID removed') - - def record_pid(self, pid_file): - pid = os.getpid() - if self.options.verbose > 1: - self.out('Writing PID %s to %s' % (pid, pid_file)) - with open(pid_file, 'w') as f: - f.write(str(pid)) - atexit.register(self._remove_pid_file, pid, pid_file, self.options.verbose) - - def stop_daemon(self): # pragma: no cover - pid_file = self.options.pid_file or 'pyramid.pid' - if not os.path.exists(pid_file): - self.out('No PID file exists in %s' % pid_file) - return 1 - pid = read_pidfile(pid_file) - if not pid: - self.out("Not a valid PID file in %s" % pid_file) - return 1 - pid = live_pidfile(pid_file) - if not pid: - self.out("PID in %s is not valid (deleting)" % pid_file) - try: - os.unlink(pid_file) - except (OSError, IOError) as e: - self.out("Could not delete: %s" % e) - return 2 - return 1 - for j in range(10): - if not live_pidfile(pid_file): - break - import signal - kill(pid, signal.SIGTERM) - time.sleep(1) - else: - self.out("failed to kill web process %s" % pid) - return 3 - if os.path.exists(pid_file): - os.unlink(pid_file) - return 0 - - def show_status(self): # pragma: no cover - pid_file = self.options.pid_file or 'pyramid.pid' - if not os.path.exists(pid_file): - self.out('No PID file %s' % pid_file) - return 1 - pid = read_pidfile(pid_file) - if not pid: - self.out('No PID in file %s' % pid_file) - return 1 - pid = live_pidfile(pid_file) - if not pid: - self.out('PID %s in %s is not running' % (pid, pid_file)) - return 1 - self.out('Server running in PID %s' % pid) - return 0 - def restart_with_reloader(self): # pragma: no cover self.restart_with_monitor(reloader=True) @@ -624,60 +301,6 @@ def restart_with_monitor(self, reloader=False): # pragma: no cover if self.options.verbose > 0: self.out('%s %s %s' % ('-' * 20, 'Restarting', '-' * 20)) - def change_user_group(self, user, group): # pragma: no cover - import pwd - import grp - - self.out('''\ -The --user and --group options have been deprecated in Pyramid 1.6. They will -be removed in a future release per Pyramid's deprecation policy. Please -consider using a real process manager for your processes like Systemd, Circus, -or Supervisor, all of which support process security. -''') - - uid = gid = None - if group: - try: - gid = int(group) - group = grp.getgrgid(gid).gr_name - except ValueError: - import grp - try: - entry = grp.getgrnam(group) - except KeyError: - raise ValueError( - "Bad group: %r; no such group exists" % group) - gid = entry.gr_gid - try: - uid = int(user) - user = pwd.getpwuid(uid).pw_name - except ValueError: - try: - entry = pwd.getpwnam(user) - except KeyError: - raise ValueError( - "Bad username: %r; no such user exists" % user) - if not gid: - gid = entry.pw_gid - uid = entry.pw_uid - if self.options.verbose > 0: - self.out('Changing user to %s:%s (%s:%s)' % ( - user, group or '(unknown)', uid, gid)) - if gid: - os.setgid(gid) - if uid: - os.setuid(uid) - - def _warn_daemon_deprecated(self): - self.out('''\ -The daemon options have been deprecated in Pyramid 1.6. They will be removed -in a future release per Pyramid's deprecation policy. Please consider using -a real process manager for your processes like Systemd, Circus, or Supervisor. - -The following commands are deprecated: - [start,stop,restart,status] --daemon, --stop-server, --status, --pid-file, --log-file -''') - class LazyWriter(object): """ @@ -718,33 +341,6 @@ def writelines(self, text): def flush(self): self.open().flush() -def live_pidfile(pidfile): # pragma: no cover - """(pidfile:str) -> int | None - Returns an int found in the named file, if there is one, - and if there is a running process with that process id. - Return None if no such process exists. - """ - pid = read_pidfile(pidfile) - if pid: - try: - kill(int(pid), 0) - return pid - except OSError as e: - if e.errno == errno.EPERM: - return pid - return None - -def read_pidfile(filename): - if os.path.exists(filename): - try: - with open(filename) as f: - content = f.read() - return int(content.strip()) - except (ValueError, IOError): - return None - else: - return None - def ensure_port_cleanup( bound_addresses, maxtries=30, sleeptime=2): # pragma: no cover """ diff --git a/pyramid/tests/test_scripts/test_pserve.py b/pyramid/tests/test_scripts/test_pserve.py index bc21665aa7..bf47636027 100644 --- a/pyramid/tests/test_scripts/test_pserve.py +++ b/pyramid/tests/test_scripts/test_pserve.py @@ -1,23 +1,11 @@ -import atexit import os import tempfile import unittest -from pyramid.compat import PY2 -if PY2: - import __builtin__ -else: - import builtins as __builtin__ - class TestPServeCommand(unittest.TestCase): def setUp(self): from pyramid.compat import NativeIO self.out_ = NativeIO() - self.pid_file = None - - def tearDown(self): - if self.pid_file and os.path.exists(self.pid_file): - os.remove(self.pid_file) def out(self, msg): self.out_.write(msg) @@ -39,172 +27,12 @@ def _makeOne(self, *args): cmd.out = self.out return cmd - def _makeOneWithPidFile(self, pid): - self.pid_file = tempfile.mktemp() - inst = self._makeOne() - with open(self.pid_file, 'w') as f: - f.write(str(pid)) - return inst - - def test_remove_pid_file_verbose(self): - inst = self._makeOneWithPidFile(os.getpid()) - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - self._assert_pid_file_removed(verbose=True) - - def test_remove_pid_file_not_verbose(self): - inst = self._makeOneWithPidFile(os.getpid()) - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=0) - self._assert_pid_file_removed(verbose=False) - - def test_remove_pid_not_a_number(self): - inst = self._makeOneWithPidFile('not a number') - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - self._assert_pid_file_removed(verbose=True) - - def test_remove_pid_current_pid_is_not_written_pid(self): - inst = self._makeOneWithPidFile(os.getpid()) - inst._remove_pid_file('99999', self.pid_file, verbosity=1) - self._assert_pid_file_not_removed('') - - def test_remove_pid_current_pid_is_not_pid_in_file(self): - inst = self._makeOneWithPidFile('99999') - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - msg = 'PID file %s contains 99999, not expected PID %s' - self._assert_pid_file_not_removed(msg % (self.pid_file, os.getpid())) - - def test_remove_pid_no_pid_file(self): - inst = self._makeOne() - self.pid_file = 'some unknown path' - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - self._assert_pid_file_removed(verbose=False) - - def test_remove_pid_file_unlink_exception(self): - inst = self._makeOneWithPidFile(os.getpid()) - self._remove_pid_unlink_exception(inst) - msg = [ - 'Removing PID file %s' % (self.pid_file), - 'Cannot remove PID file: (Some OSError - unlink)', - 'Stale PID removed'] - self._assert_pid_file_not_removed(msg=''.join(msg)) - with open(self.pid_file) as f: - self.assertEqual(f.read(), '') - - def test_remove_pid_file_stale_pid_write_exception(self): - inst = self._makeOneWithPidFile(os.getpid()) - self._remove_pid_unlink_and_write_exceptions(inst) - msg = [ - 'Removing PID file %s' % (self.pid_file), - 'Cannot remove PID file: (Some OSError - unlink)', - 'Stale PID left in file: %s ' % (self.pid_file), - '(Some OSError - open)'] - self._assert_pid_file_not_removed(msg=''.join(msg)) - with open(self.pid_file) as f: - self.assertEqual(int(f.read()), os.getpid()) - - def test_record_pid_verbose(self): - self._assert_record_pid(verbosity=2, msg='Writing PID %d to %s') - - def test_record_pid_not_verbose(self): - self._assert_record_pid(verbosity=1, msg='') - - def _remove_pid_unlink_exception(self, inst): - old_unlink = os.unlink - def fake_unlink(filename): - raise OSError('Some OSError - unlink') - - try: - os.unlink = fake_unlink - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - finally: - os.unlink = old_unlink - - def _remove_pid_unlink_and_write_exceptions(self, inst): - old_unlink = os.unlink - def fake_unlink(filename): - raise OSError('Some OSError - unlink') - - run_already = [] - old_open = __builtin__.open - def fake_open(*args): - if not run_already: - run_already.append(True) - return old_open(*args) - raise OSError('Some OSError - open') - - try: - os.unlink = fake_unlink - __builtin__.open = fake_open - inst._remove_pid_file(os.getpid(), self.pid_file, verbosity=1) - finally: - os.unlink = old_unlink - __builtin__.open = old_open - - def _assert_pid_file_removed(self, verbose=False): - self.assertFalse(os.path.exists(self.pid_file)) - msg = 'Removing PID file %s' % (self.pid_file) if verbose else '' - self.assertEqual(self.out_.getvalue(), msg) - - def _assert_pid_file_not_removed(self, msg): - self.assertTrue(os.path.exists(self.pid_file)) - self.assertEqual(self.out_.getvalue(), msg) - - def _assert_record_pid(self, verbosity, msg): - old_atexit = atexit.register - def fake_atexit(*args): - pass - - self.pid_file = tempfile.mktemp() - pid = os.getpid() - inst = self._makeOne() - inst.options.verbose = verbosity - - try: - atexit.register = fake_atexit - inst.record_pid(self.pid_file) - finally: - atexit.register = old_atexit - - msg = msg % (pid, self.pid_file) if msg else '' - self.assertEqual(self.out_.getvalue(), msg) - with open(self.pid_file) as f: - self.assertEqual(int(f.read()), pid) - def test_run_no_args(self): inst = self._makeOne() result = inst.run() self.assertEqual(result, 2) self.assertEqual(self.out_.getvalue(), 'You must give a config file') - def test_run_stop_daemon_no_such_pid_file(self): - path = os.path.join(os.path.dirname(__file__), 'wontexist.pid') - inst = self._makeOne('--stop-daemon', '--pid-file=%s' % path) - inst.run() - msg = 'No PID file exists in %s' % path - self.assertTrue(msg in self.out_.getvalue()) - - def test_run_stop_daemon_bad_pid_file(self): - path = __file__ - inst = self._makeOne('--stop-daemon', '--pid-file=%s' % path) - inst.run() - msg = 'Not a valid PID file in %s' % path - self.assertTrue(msg in self.out_.getvalue()) - - def test_run_stop_daemon_invalid_pid_in_file(self): - fn = tempfile.mktemp() - with open(fn, 'wb') as tmp: - tmp.write(b'9999999') - tmp.close() - inst = self._makeOne('--stop-daemon', '--pid-file=%s' % fn) - inst.run() - msg = 'PID in %s is not valid (deleting)' % fn - self.assertTrue(msg in self.out_.getvalue()) - - def test_get_options_with_command(self): - inst = self._makeOne() - inst.args = ['foo', 'stop', 'a=1', 'b=2'] - result = inst.get_options() - self.assertEqual(result, {'a': '1', 'b': '2'}) - def test_get_options_no_command(self): inst = self._makeOne() inst.args = ['foo', 'a=1', 'b=2'] @@ -233,29 +61,6 @@ def test_parse_vars_bad(self): inst.loadserver = self._get_server self.assertRaises(ValueError, inst.run) -class Test_read_pidfile(unittest.TestCase): - def _callFUT(self, filename): - from pyramid.scripts.pserve import read_pidfile - return read_pidfile(filename) - - def test_read_pidfile(self): - filename = tempfile.mktemp() - try: - with open(filename, 'w') as f: - f.write('12345') - result = self._callFUT(filename) - self.assertEqual(result, 12345) - finally: - os.remove(filename) - - def test_read_pidfile_no_pid_file(self): - result = self._callFUT('some unknown path') - self.assertEqual(result, None) - - def test_read_pidfile_not_a_number(self): - result = self._callFUT(__file__) - self.assertEqual(result, None) - class Test_main(unittest.TestCase): def _callFUT(self, argv): from pyramid.scripts.pserve import main