diff options
Diffstat (limited to 'cherrypy')
117 files changed, 28745 insertions, 0 deletions
diff --git a/cherrypy/LICENSE.txt b/cherrypy/LICENSE.txt new file mode 100644 index 0000000..8db13fb --- /dev/null +++ b/cherrypy/LICENSE.txt @@ -0,0 +1,25 @@ +Copyright (c) 2004-2011, CherryPy Team (team@cherrypy.org)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the CherryPy Team nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cherrypy/__init__.py b/cherrypy/__init__.py new file mode 100755 index 0000000..eb7cabf --- /dev/null +++ b/cherrypy/__init__.py @@ -0,0 +1,620 @@ +"""CherryPy is a pythonic, object-oriented HTTP framework. + + +CherryPy consists of not one, but four separate API layers. + +The APPLICATION LAYER is the simplest. CherryPy applications are written as +a tree of classes and methods, where each branch in the tree corresponds to +a branch in the URL path. Each method is a 'page handler', which receives +GET and POST params as keyword arguments, and returns or yields the (HTML) +body of the response. The special method name 'index' is used for paths +that end in a slash, and the special method name 'default' is used to +handle multiple paths via a single handler. This layer also includes: + + * the 'exposed' attribute (and cherrypy.expose) + * cherrypy.quickstart() + * _cp_config attributes + * cherrypy.tools (including cherrypy.session) + * cherrypy.url() + +The ENVIRONMENT LAYER is used by developers at all levels. It provides +information about the current request and response, plus the application +and server environment, via a (default) set of top-level objects: + + * cherrypy.request + * cherrypy.response + * cherrypy.engine + * cherrypy.server + * cherrypy.tree + * cherrypy.config + * cherrypy.thread_data + * cherrypy.log + * cherrypy.HTTPError, NotFound, and HTTPRedirect + * cherrypy.lib + +The EXTENSION LAYER allows advanced users to construct and share their own +plugins. It consists of: + + * Hook API + * Tool API + * Toolbox API + * Dispatch API + * Config Namespace API + +Finally, there is the CORE LAYER, which uses the core API's to construct +the default components which are available at higher layers. You can think +of the default components as the 'reference implementation' for CherryPy. +Megaframeworks (and advanced users) may replace the default components +with customized or extended components. The core API's are: + + * Application API + * Engine API + * Request API + * Server API + * WSGI API + +These API's are described in the CherryPy specification: +http://www.cherrypy.org/wiki/CherryPySpec +""" + +__version__ = "3.2.0" + +from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode +from cherrypy._cpcompat import basestring, unicodestr + +from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect +from cherrypy._cperror import NotFound, CherryPyException, TimeoutError + +from cherrypy import _cpdispatch as dispatch + +from cherrypy import _cptools +tools = _cptools.default_toolbox +Tool = _cptools.Tool + +from cherrypy import _cprequest +from cherrypy.lib import httputil as _httputil + +from cherrypy import _cptree +tree = _cptree.Tree() +from cherrypy._cptree import Application +from cherrypy import _cpwsgi as wsgi + +from cherrypy import process +try: + from cherrypy.process import win32 + engine = win32.Win32Bus() + engine.console_control_handler = win32.ConsoleCtrlHandler(engine) + del win32 +except ImportError: + engine = process.bus + + +# Timeout monitor +class _TimeoutMonitor(process.plugins.Monitor): + + def __init__(self, bus): + self.servings = [] + process.plugins.Monitor.__init__(self, bus, self.run) + + def acquire(self): + self.servings.append((serving.request, serving.response)) + + def release(self): + try: + self.servings.remove((serving.request, serving.response)) + except ValueError: + pass + + def run(self): + """Check timeout on all responses. (Internal)""" + for req, resp in self.servings: + resp.check_timeout() +engine.timeout_monitor = _TimeoutMonitor(engine) +engine.timeout_monitor.subscribe() + +engine.autoreload = process.plugins.Autoreloader(engine) +engine.autoreload.subscribe() + +engine.thread_manager = process.plugins.ThreadManager(engine) +engine.thread_manager.subscribe() + +engine.signal_handler = process.plugins.SignalHandler(engine) + + +from cherrypy import _cpserver +server = _cpserver.Server() +server.subscribe() + + +def quickstart(root=None, script_name="", config=None): + """Mount the given root, start the builtin server (and engine), then block. + + root: an instance of a "controller class" (a collection of page handler + methods) which represents the root of the application. + script_name: a string containing the "mount point" of the application. + This should start with a slash, and be the path portion of the URL + at which to mount the given root. For example, if root.index() will + handle requests to "http://www.example.com:8080/dept/app1/", then + the script_name argument would be "/dept/app1". + + It MUST NOT end in a slash. If the script_name refers to the root + of the URI, it MUST be an empty string (not "/"). + config: a file or dict containing application config. If this contains + a [global] section, those entries will be used in the global + (site-wide) config. + """ + if config: + _global_conf_alias.update(config) + + tree.mount(root, script_name, config) + + if hasattr(engine, "signal_handler"): + engine.signal_handler.subscribe() + if hasattr(engine, "console_control_handler"): + engine.console_control_handler.subscribe() + + engine.start() + engine.block() + + +from cherrypy._cpcompat import threadlocal as _local + +class _Serving(_local): + """An interface for registering request and response objects. + + Rather than have a separate "thread local" object for the request and + the response, this class works as a single threadlocal container for + both objects (and any others which developers wish to define). In this + way, we can easily dump those objects when we stop/start a new HTTP + conversation, yet still refer to them as module-level globals in a + thread-safe way. + """ + + request = _cprequest.Request(_httputil.Host("127.0.0.1", 80), + _httputil.Host("127.0.0.1", 1111)) + """ + The request object for the current thread. In the main thread, + and any threads which are not receiving HTTP requests, this is None.""" + + response = _cprequest.Response() + """ + The response object for the current thread. In the main thread, + and any threads which are not receiving HTTP requests, this is None.""" + + def load(self, request, response): + self.request = request + self.response = response + + def clear(self): + """Remove all attributes of self.""" + self.__dict__.clear() + +serving = _Serving() + + +class _ThreadLocalProxy(object): + + __slots__ = ['__attrname__', '__dict__'] + + def __init__(self, attrname): + self.__attrname__ = attrname + + def __getattr__(self, name): + child = getattr(serving, self.__attrname__) + return getattr(child, name) + + def __setattr__(self, name, value): + if name in ("__attrname__", ): + object.__setattr__(self, name, value) + else: + child = getattr(serving, self.__attrname__) + setattr(child, name, value) + + def __delattr__(self, name): + child = getattr(serving, self.__attrname__) + delattr(child, name) + + def _get_dict(self): + child = getattr(serving, self.__attrname__) + d = child.__class__.__dict__.copy() + d.update(child.__dict__) + return d + __dict__ = property(_get_dict) + + def __getitem__(self, key): + child = getattr(serving, self.__attrname__) + return child[key] + + def __setitem__(self, key, value): + child = getattr(serving, self.__attrname__) + child[key] = value + + def __delitem__(self, key): + child = getattr(serving, self.__attrname__) + del child[key] + + def __contains__(self, key): + child = getattr(serving, self.__attrname__) + return key in child + + def __len__(self): + child = getattr(serving, self.__attrname__) + return len(child) + + def __nonzero__(self): + child = getattr(serving, self.__attrname__) + return bool(child) + # Python 3 + __bool__ = __nonzero__ + +# Create request and response object (the same objects will be used +# throughout the entire life of the webserver, but will redirect +# to the "serving" object) +request = _ThreadLocalProxy('request') +response = _ThreadLocalProxy('response') + +# Create thread_data object as a thread-specific all-purpose storage +class _ThreadData(_local): + """A container for thread-specific data.""" +thread_data = _ThreadData() + + +# Monkeypatch pydoc to allow help() to go through the threadlocal proxy. +# Jan 2007: no Googleable examples of anyone else replacing pydoc.resolve. +# The only other way would be to change what is returned from type(request) +# and that's not possible in pure Python (you'd have to fake ob_type). +def _cherrypy_pydoc_resolve(thing, forceload=0): + """Given an object or a path to an object, get the object and its name.""" + if isinstance(thing, _ThreadLocalProxy): + thing = getattr(serving, thing.__attrname__) + return _pydoc._builtin_resolve(thing, forceload) + +try: + import pydoc as _pydoc + _pydoc._builtin_resolve = _pydoc.resolve + _pydoc.resolve = _cherrypy_pydoc_resolve +except ImportError: + pass + + +from cherrypy import _cplogging + +class _GlobalLogManager(_cplogging.LogManager): + """A site-wide LogManager; routes to app.log or global log as appropriate. + + This :class:`LogManager<cherrypy._cplogging.LogManager>` implements + cherrypy.log() and cherrypy.log.access(). If either + function is called during a request, the message will be sent to the + logger for the current Application. If they are called outside of a + request, the message will be sent to the site-wide logger. + """ + + def __call__(self, *args, **kwargs): + """Log the given message to the app.log or global log as appropriate.""" + # Do NOT use try/except here. See http://www.cherrypy.org/ticket/945 + if hasattr(request, 'app') and hasattr(request.app, 'log'): + log = request.app.log + else: + log = self + return log.error(*args, **kwargs) + + def access(self): + """Log an access message to the app.log or global log as appropriate.""" + try: + return request.app.log.access() + except AttributeError: + return _cplogging.LogManager.access(self) + + +log = _GlobalLogManager() +# Set a default screen handler on the global log. +log.screen = True +log.error_file = '' +# Using an access file makes CP about 10% slower. Leave off by default. +log.access_file = '' + +def _buslog(msg, level): + log.error(msg, 'ENGINE', severity=level) +engine.subscribe('log', _buslog) + +# Helper functions for CP apps # + + +def expose(func=None, alias=None): + """Expose the function, optionally providing an alias or set of aliases.""" + def expose_(func): + func.exposed = True + if alias is not None: + if isinstance(alias, basestring): + parents[alias.replace(".", "_")] = func + else: + for a in alias: + parents[a.replace(".", "_")] = func + return func + + import sys, types + if isinstance(func, (types.FunctionType, types.MethodType)): + if alias is None: + # @expose + func.exposed = True + return func + else: + # func = expose(func, alias) + parents = sys._getframe(1).f_locals + return expose_(func) + elif func is None: + if alias is None: + # @expose() + parents = sys._getframe(1).f_locals + return expose_ + else: + # @expose(alias="alias") or + # @expose(alias=["alias1", "alias2"]) + parents = sys._getframe(1).f_locals + return expose_ + else: + # @expose("alias") or + # @expose(["alias1", "alias2"]) + parents = sys._getframe(1).f_locals + alias = func + return expose_ + +def popargs(*args, **kwargs): + """A decorator for _cp_dispatch + (cherrypy.dispatch.Dispatcher.dispatch_method_name). + + Optional keyword argument: handler=(Object or Function) + + Provides a _cp_dispatch function that pops off path segments into + cherrypy.request.params under the names specified. The dispatch + is then forwarded on to the next vpath element. + + Note that any existing (and exposed) member function of the class that + popargs is applied to will override that value of the argument. For + instance, if you have a method named "list" on the class decorated with + popargs, then accessing "/list" will call that function instead of popping + it off as the requested parameter. This restriction applies to all + _cp_dispatch functions. The only way around this restriction is to create + a "blank class" whose only function is to provide _cp_dispatch. + + If there are path elements after the arguments, or more arguments + are requested than are available in the vpath, then the 'handler' + keyword argument specifies the next object to handle the parameterized + request. If handler is not specified or is None, then self is used. + If handler is a function rather than an instance, then that function + will be called with the args specified and the return value from that + function used as the next object INSTEAD of adding the parameters to + cherrypy.request.args. + + This decorator may be used in one of two ways: + + As a class decorator: + @cherrypy.popargs('year', 'month', 'day') + class Blog: + def index(self, year=None, month=None, day=None): + #Process the parameters here; any url like + #/, /2009, /2009/12, or /2009/12/31 + #will fill in the appropriate parameters. + + def create(self): + #This link will still be available at /create. Defined functions + #take precedence over arguments. + + Or as a member of a class: + class Blog: + _cp_dispatch = cherrypy.popargs('year', 'month', 'day') + #... + + The handler argument may be used to mix arguments with built in functions. + For instance, the following setup allows different activities at the + day, month, and year level: + + class DayHandler: + def index(self, year, month, day): + #Do something with this day; probably list entries + + def delete(self, year, month, day): + #Delete all entries for this day + + @cherrypy.popargs('day', handler=DayHandler()) + class MonthHandler: + def index(self, year, month): + #Do something with this month; probably list entries + + def delete(self, year, month): + #Delete all entries for this month + + @cherrypy.popargs('month', handler=MonthHandler()) + class YearHandler: + def index(self, year): + #Do something with this year + + #... + + @cherrypy.popargs('year', handler=YearHandler()) + class Root: + def index(self): + #... + + """ + + #Since keyword arg comes after *args, we have to process it ourselves + #for lower versions of python. + + handler = None + handler_call = False + for k,v in kwargs.items(): + if k == 'handler': + handler = v + else: + raise TypeError( + "cherrypy.popargs() got an unexpected keyword argument '{0}'" \ + .format(k) + ) + + import inspect + + if handler is not None \ + and (hasattr(handler, '__call__') or inspect.isclass(handler)): + handler_call = True + + def decorated(cls_or_self=None, vpath=None): + if inspect.isclass(cls_or_self): + #cherrypy.popargs is a class decorator + cls = cls_or_self + setattr(cls, dispatch.Dispatcher.dispatch_method_name, decorated) + return cls + + #We're in the actual function + self = cls_or_self + parms = {} + for arg in args: + if not vpath: + break + parms[arg] = vpath.pop(0) + + if handler is not None: + if handler_call: + return handler(**parms) + else: + request.params.update(parms) + return handler + + request.params.update(parms) + + #If we are the ultimate handler, then to prevent our _cp_dispatch + #from being called again, we will resolve remaining elements through + #getattr() directly. + if vpath: + return getattr(self, vpath.pop(0), None) + else: + return self + + return decorated + +def url(path="", qs="", script_name=None, base=None, relative=None): + """Create an absolute URL for the given path. + + If 'path' starts with a slash ('/'), this will return + (base + script_name + path + qs). + If it does not start with a slash, this returns + (base + script_name [+ request.path_info] + path + qs). + + If script_name is None, cherrypy.request will be used + to find a script_name, if available. + + If base is None, cherrypy.request.base will be used (if available). + Note that you can use cherrypy.tools.proxy to change this. + + Finally, note that this function can be used to obtain an absolute URL + for the current request path (minus the querystring) by passing no args. + If you call url(qs=cherrypy.request.query_string), you should get the + original browser URL (assuming no internal redirections). + + If relative is None or not provided, request.app.relative_urls will + be used (if available, else False). If False, the output will be an + absolute URL (including the scheme, host, vhost, and script_name). + If True, the output will instead be a URL that is relative to the + current request path, perhaps including '..' atoms. If relative is + the string 'server', the output will instead be a URL that is + relative to the server root; i.e., it will start with a slash. + """ + if isinstance(qs, (tuple, list, dict)): + qs = _urlencode(qs) + if qs: + qs = '?' + qs + + if request.app: + if not path.startswith("/"): + # Append/remove trailing slash from path_info as needed + # (this is to support mistyped URL's without redirecting; + # if you want to redirect, use tools.trailing_slash). + pi = request.path_info + if request.is_index is True: + if not pi.endswith('/'): + pi = pi + '/' + elif request.is_index is False: + if pi.endswith('/') and pi != '/': + pi = pi[:-1] + + if path == "": + path = pi + else: + path = _urljoin(pi, path) + + if script_name is None: + script_name = request.script_name + if base is None: + base = request.base + + newurl = base + script_name + path + qs + else: + # No request.app (we're being called outside a request). + # We'll have to guess the base from server.* attributes. + # This will produce very different results from the above + # if you're using vhosts or tools.proxy. + if base is None: + base = server.base() + + path = (script_name or "") + path + newurl = base + path + qs + + if './' in newurl: + # Normalize the URL by removing ./ and ../ + atoms = [] + for atom in newurl.split('/'): + if atom == '.': + pass + elif atom == '..': + atoms.pop() + else: + atoms.append(atom) + newurl = '/'.join(atoms) + + # At this point, we should have a fully-qualified absolute URL. + + if relative is None: + relative = getattr(request.app, "relative_urls", False) + + # See http://www.ietf.org/rfc/rfc2396.txt + if relative == 'server': + # "A relative reference beginning with a single slash character is + # termed an absolute-path reference, as defined by <abs_path>..." + # This is also sometimes called "server-relative". + newurl = '/' + '/'.join(newurl.split('/', 3)[3:]) + elif relative: + # "A relative reference that does not begin with a scheme name + # or a slash character is termed a relative-path reference." + old = url().split('/')[:-1] + new = newurl.split('/') + while old and new: + a, b = old[0], new[0] + if a != b: + break + old.pop(0) + new.pop(0) + new = (['..'] * len(old)) + new + newurl = '/'.join(new) + + return newurl + + +# import _cpconfig last so it can reference other top-level objects +from cherrypy import _cpconfig +# Use _global_conf_alias so quickstart can use 'config' as an arg +# without shadowing cherrypy.config. +config = _global_conf_alias = _cpconfig.Config() +config.defaults = { + 'tools.log_tracebacks.on': True, + 'tools.log_headers.on': True, + 'tools.trailing_slash.on': True, + 'tools.encode.on': True + } +config.namespaces["log"] = lambda k, v: setattr(log, k, v) +config.namespaces["checker"] = lambda k, v: setattr(checker, k, v) +# Must reset to get our defaults applied. +config.reset() + +from cherrypy import _cpchecker +checker = _cpchecker.Checker() +engine.subscribe('start', checker) diff --git a/cherrypy/_cpchecker.py b/cherrypy/_cpchecker.py new file mode 100755 index 0000000..7ccfd89 --- /dev/null +++ b/cherrypy/_cpchecker.py @@ -0,0 +1,327 @@ +import os +import warnings + +import cherrypy +from cherrypy._cpcompat import iteritems, copykeys, builtins + + +class Checker(object): + """A checker for CherryPy sites and their mounted applications. + + When this object is called at engine startup, it executes each + of its own methods whose names start with ``check_``. If you wish + to disable selected checks, simply add a line in your global + config which sets the appropriate method to False:: + + [global] + checker.check_skipped_app_config = False + + You may also dynamically add or replace ``check_*`` methods in this way. + """ + + on = True + """If True (the default), run all checks; if False, turn off all checks.""" + + + def __init__(self): + self._populate_known_types() + + def __call__(self): + """Run all check_* methods.""" + if self.on: + oldformatwarning = warnings.formatwarning + warnings.formatwarning = self.formatwarning + try: + for name in dir(self): + if name.startswith("check_"): + method = getattr(self, name) + if method and hasattr(method, '__call__'): + method() + finally: + warnings.formatwarning = oldformatwarning + + def formatwarning(self, message, category, filename, lineno, line=None): + """Function to format a warning.""" + return "CherryPy Checker:\n%s\n\n" % message + + # This value should be set inside _cpconfig. + global_config_contained_paths = False + + def check_app_config_entries_dont_start_with_script_name(self): + """Check for Application config with sections that repeat script_name.""" + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + if not app.config: + continue + if sn == '': + continue + sn_atoms = sn.strip("/").split("/") + for key in app.config.keys(): + key_atoms = key.strip("/").split("/") + if key_atoms[:len(sn_atoms)] == sn_atoms: + warnings.warn( + "The application mounted at %r has config " \ + "entries that start with its script name: %r" % (sn, key)) + + def check_site_config_entries_in_app_config(self): + """Check for mounted Applications that have site-scoped config.""" + for sn, app in iteritems(cherrypy.tree.apps): + if not isinstance(app, cherrypy.Application): + continue + + msg = [] + for section, entries in iteritems(app.config): + if section.startswith('/'): + for key, value in iteritems(entries): + for n in ("engine.", "server.", "tree.", "checker."): + if key.startswith(n): + msg.append("[%s] %s = %s" % (section, key, value)) + if msg: + msg.insert(0, + "The application mounted at %r contains the following " + "config entries, which are only allowed in site-wide " + "config. Move them to a [global] section and pass them " + "to cherrypy.config.update() instead of tree.mount()." % sn) + warnings.warn(os.linesep.join(msg)) + + def check_skipped_app_config(self): + """Check for mounted Applications that have no config.""" + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + if not app.config: + msg = "The Application mounted at %r has an empty config." % sn + if self.global_config_contained_paths: + msg += (" It looks like the config you passed to " + "cherrypy.config.update() contains application-" + "specific sections. You must explicitly pass " + "application config via " + "cherrypy.tree.mount(..., config=app_config)") + warnings.warn(msg) + return + + def check_app_config_brackets(self): + """Check for Application config with extraneous brackets in section names.""" + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + if not app.config: + continue + for key in app.config.keys(): + if key.startswith("[") or key.endswith("]"): + warnings.warn( + "The application mounted at %r has config " \ + "section names with extraneous brackets: %r. " + "Config *files* need brackets; config *dicts* " + "(e.g. passed to tree.mount) do not." % (sn, key)) + + def check_static_paths(self): + """Check Application config for incorrect static paths.""" + # Use the dummy Request object in the main thread. + request = cherrypy.request + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + request.app = app + for section in app.config: + # get_resource will populate request.config + request.get_resource(section + "/dummy.html") + conf = request.config.get + + if conf("tools.staticdir.on", False): + msg = "" + root = conf("tools.staticdir.root") + dir = conf("tools.staticdir.dir") + if dir is None: + msg = "tools.staticdir.dir is not set." + else: + fulldir = "" + if os.path.isabs(dir): + fulldir = dir + if root: + msg = ("dir is an absolute path, even " + "though a root is provided.") + testdir = os.path.join(root, dir[1:]) + if os.path.exists(testdir): + msg += ("\nIf you meant to serve the " + "filesystem folder at %r, remove " + "the leading slash from dir." % testdir) + else: + if not root: + msg = "dir is a relative path and no root provided." + else: + fulldir = os.path.join(root, dir) + if not os.path.isabs(fulldir): + msg = "%r is not an absolute path." % fulldir + + if fulldir and not os.path.exists(fulldir): + if msg: + msg += "\n" + msg += ("%r (root + dir) is not an existing " + "filesystem path." % fulldir) + + if msg: + warnings.warn("%s\nsection: [%s]\nroot: %r\ndir: %r" + % (msg, section, root, dir)) + + + # -------------------------- Compatibility -------------------------- # + + obsolete = { + 'server.default_content_type': 'tools.response_headers.headers', + 'log_access_file': 'log.access_file', + 'log_config_options': None, + 'log_file': 'log.error_file', + 'log_file_not_found': None, + 'log_request_headers': 'tools.log_headers.on', + 'log_to_screen': 'log.screen', + 'show_tracebacks': 'request.show_tracebacks', + 'throw_errors': 'request.throw_errors', + 'profiler.on': ('cherrypy.tree.mount(profiler.make_app(' + 'cherrypy.Application(Root())))'), + } + + deprecated = {} + + def _compat(self, config): + """Process config and warn on each obsolete or deprecated entry.""" + for section, conf in config.items(): + if isinstance(conf, dict): + for k, v in conf.items(): + if k in self.obsolete: + warnings.warn("%r is obsolete. Use %r instead.\n" + "section: [%s]" % + (k, self.obsolete[k], section)) + elif k in self.deprecated: + warnings.warn("%r is deprecated. Use %r instead.\n" + "section: [%s]" % + (k, self.deprecated[k], section)) + else: + if section in self.obsolete: + warnings.warn("%r is obsolete. Use %r instead." + % (section, self.obsolete[section])) + elif section in self.deprecated: + warnings.warn("%r is deprecated. Use %r instead." + % (section, self.deprecated[section])) + + def check_compatibility(self): + """Process config and warn on each obsolete or deprecated entry.""" + self._compat(cherrypy.config) + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + self._compat(app.config) + + + # ------------------------ Known Namespaces ------------------------ # + + extra_config_namespaces = [] + + def _known_ns(self, app): + ns = ["wsgi"] + ns.extend(copykeys(app.toolboxes)) + ns.extend(copykeys(app.namespaces)) + ns.extend(copykeys(app.request_class.namespaces)) + ns.extend(copykeys(cherrypy.config.namespaces)) + ns += self.extra_config_namespaces + + for section, conf in app.config.items(): + is_path_section = section.startswith("/") + if is_path_section and isinstance(conf, dict): + for k, v in conf.items(): + atoms = k.split(".") + if len(atoms) > 1: + if atoms[0] not in ns: + # Spit out a special warning if a known + # namespace is preceded by "cherrypy." + if (atoms[0] == "cherrypy" and atoms[1] in ns): + msg = ("The config entry %r is invalid; " + "try %r instead.\nsection: [%s]" + % (k, ".".join(atoms[1:]), section)) + else: + msg = ("The config entry %r is invalid, because " + "the %r config namespace is unknown.\n" + "section: [%s]" % (k, atoms[0], section)) + warnings.warn(msg) + elif atoms[0] == "tools": + if atoms[1] not in dir(cherrypy.tools): + msg = ("The config entry %r may be invalid, " + "because the %r tool was not found.\n" + "section: [%s]" % (k, atoms[1], section)) + warnings.warn(msg) + + def check_config_namespaces(self): + """Process config and warn on each unknown config namespace.""" + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + self._known_ns(app) + + + + + # -------------------------- Config Types -------------------------- # + + known_config_types = {} + + def _populate_known_types(self): + b = [x for x in vars(builtins).values() + if type(x) is type(str)] + + def traverse(obj, namespace): + for name in dir(obj): + # Hack for 3.2's warning about body_params + if name == 'body_params': + continue + vtype = type(getattr(obj, name, None)) + if vtype in b: + self.known_config_types[namespace + "." + name] = vtype + + traverse(cherrypy.request, "request") + traverse(cherrypy.response, "response") + traverse(cherrypy.server, "server") + traverse(cherrypy.engine, "engine") + traverse(cherrypy.log, "log") + + def _known_types(self, config): + msg = ("The config entry %r in section %r is of type %r, " + "which does not match the expected type %r.") + + for section, conf in config.items(): + if isinstance(conf, dict): + for k, v in conf.items(): + if v is not None: + expected_type = self.known_config_types.get(k, None) + vtype = type(v) + if expected_type and vtype != expected_type: + warnings.warn(msg % (k, section, vtype.__name__, + expected_type.__name__)) + else: + k, v = section, conf + if v is not None: + expected_type = self.known_config_types.get(k, None) + vtype = type(v) + if expected_type and vtype != expected_type: + warnings.warn(msg % (k, section, vtype.__name__, + expected_type.__name__)) + + def check_config_types(self): + """Assert that config values are of the same type as default values.""" + self._known_types(cherrypy.config) + for sn, app in cherrypy.tree.apps.items(): + if not isinstance(app, cherrypy.Application): + continue + self._known_types(app.config) + + + # -------------------- Specific config warnings -------------------- # + + def check_localhost(self): + """Warn if any socket_host is 'localhost'. See #711.""" + for k, v in cherrypy.config.items(): + if k == 'server.socket_host' and v == 'localhost': + warnings.warn("The use of 'localhost' as a socket host can " + "cause problems on newer systems, since 'localhost' can " + "map to either an IPv4 or an IPv6 address. You should " + "use '127.0.0.1' or '[::1]' instead.") diff --git a/cherrypy/_cpcompat.py b/cherrypy/_cpcompat.py new file mode 100755 index 0000000..216dddd --- /dev/null +++ b/cherrypy/_cpcompat.py @@ -0,0 +1,283 @@ +"""Compatibility code for using CherryPy with various versions of Python. + +CherryPy 3.2 is compatible with Python versions 2.3+. This module provides a +useful abstraction over the differences between Python versions, sometimes by +preferring a newer idiom, sometimes an older one, and sometimes a custom one. + +In particular, Python 2 uses str and '' for byte strings, while Python 3 +uses str and '' for unicode strings. We will call each of these the 'native +string' type for each version. Because of this major difference, this module +provides new 'bytestr', 'unicodestr', and 'nativestr' attributes, as well as +two functions: 'ntob', which translates native strings (of type 'str') into +byte strings regardless of Python version, and 'ntou', which translates native +strings to unicode strings. This also provides a 'BytesIO' name for dealing +specifically with bytes, and a 'StringIO' name for dealing with native strings. +It also provides a 'base64_decode' function with native strings as input and +output. +""" +import os +import sys + +if sys.version_info >= (3, 0): + bytestr = bytes + unicodestr = str + nativestr = unicodestr + basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 3, the native string type is unicode + return n.encode(encoding) + def ntou(n, encoding='ISO-8859-1'): + """Return the given native string as a unicode string with the given encoding.""" + # In Python 3, the native string type is unicode + return n + # type("") + from io import StringIO + # bytes: + from io import BytesIO as BytesIO +else: + # Python 2 + bytestr = str + unicodestr = unicode + nativestr = bytestr + basestring = basestring + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 2, the native string type is bytes. Assume it's already + # in the given encoding, which for ISO-8859-1 is almost always what + # was intended. + return n + def ntou(n, encoding='ISO-8859-1'): + """Return the given native string as a unicode string with the given encoding.""" + # In Python 2, the native string type is bytes. Assume it's already + # in the given encoding, which for ISO-8859-1 is almost always what + # was intended. + return n.decode(encoding) + try: + # type("") + from cStringIO import StringIO + except ImportError: + # type("") + from StringIO import StringIO + # bytes: + BytesIO = StringIO + +try: + set = set +except NameError: + from sets import Set as set + +try: + # Python 3.1+ + from base64 import decodebytes as _base64_decodebytes +except ImportError: + # Python 3.0- + # since CherryPy claims compability with Python 2.3, we must use + # the legacy API of base64 + from base64 import decodestring as _base64_decodebytes + +def base64_decode(n, encoding='ISO-8859-1'): + """Return the native string base64-decoded (as a native string).""" + if isinstance(n, unicodestr): + b = n.encode(encoding) + else: + b = n + b = _base64_decodebytes(b) + if nativestr is unicodestr: + return b.decode(encoding) + else: + return b + +try: + # Python 2.5+ + from hashlib import md5 +except ImportError: + from md5 import new as md5 + +try: + # Python 2.5+ + from hashlib import sha1 as sha +except ImportError: + from sha import new as sha + +try: + sorted = sorted +except NameError: + def sorted(i): + i = i[:] + i.sort() + return i + +try: + reversed = reversed +except NameError: + def reversed(x): + i = len(x) + while i > 0: + i -= 1 + yield x[i] + +try: + # Python 3 + from urllib.parse import urljoin, urlencode + from urllib.parse import quote, quote_plus + from urllib.request import unquote, urlopen + from urllib.request import parse_http_list, parse_keqv_list +except ImportError: + # Python 2 + from urlparse import urljoin + from urllib import urlencode, urlopen + from urllib import quote, quote_plus + from urllib import unquote + from urllib2 import parse_http_list, parse_keqv_list + +try: + from threading import local as threadlocal +except ImportError: + from cherrypy._cpthreadinglocal import local as threadlocal + +try: + dict.iteritems + # Python 2 + iteritems = lambda d: d.iteritems() + copyitems = lambda d: d.items() +except AttributeError: + # Python 3 + iteritems = lambda d: d.items() + copyitems = lambda d: list(d.items()) + +try: + dict.iterkeys + # Python 2 + iterkeys = lambda d: d.iterkeys() + copykeys = lambda d: d.keys() +except AttributeError: + # Python 3 + iterkeys = lambda d: d.keys() + copykeys = lambda d: list(d.keys()) + +try: + dict.itervalues + # Python 2 + itervalues = lambda d: d.itervalues() + copyvalues = lambda d: d.values() +except AttributeError: + # Python 3 + itervalues = lambda d: d.values() + copyvalues = lambda d: list(d.values()) + +try: + # Python 3 + import builtins +except ImportError: + # Python 2 + import __builtin__ as builtins + +try: + # Python 2. We have to do it in this order so Python 2 builds + # don't try to import the 'http' module from cherrypy.lib + from Cookie import SimpleCookie, CookieError + from httplib import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected + from BaseHTTPServer import BaseHTTPRequestHandler +except ImportError: + # Python 3 + from http.cookies import SimpleCookie, CookieError + from http.client import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected + from http.server import BaseHTTPRequestHandler + +try: + # Python 2 + xrange = xrange +except NameError: + # Python 3 + xrange = range + +import threading +if hasattr(threading.Thread, "daemon"): + # Python 2.6+ + def get_daemon(t): + return t.daemon + def set_daemon(t, val): + t.daemon = val +else: + def get_daemon(t): + return t.isDaemon() + def set_daemon(t, val): + t.setDaemon(val) + +try: + from email.utils import formatdate + def HTTPDate(timeval=None): + return formatdate(timeval, usegmt=True) +except ImportError: + from rfc822 import formatdate as HTTPDate + +try: + # Python 3 + from urllib.parse import unquote as parse_unquote + def unquote_qs(atom, encoding, errors='strict'): + return parse_unquote(atom.replace('+', ' '), encoding=encoding, errors=errors) +except ImportError: + # Python 2 + from urllib import unquote as parse_unquote + def unquote_qs(atom, encoding, errors='strict'): + return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) + +try: + # Prefer simplejson, which is usually more advanced than the builtin module. + import simplejson as json + json_decode = json.JSONDecoder().decode + json_encode = json.JSONEncoder().iterencode +except ImportError: + if sys.version_info >= (3, 0): + # Python 3.0: json is part of the standard library, + # but outputs unicode. We need bytes. + import json + json_decode = json.JSONDecoder().decode + _json_encode = json.JSONEncoder().iterencode + def json_encode(value): + for chunk in _json_encode(value): + yield chunk.encode('utf8') + elif sys.version_info >= (2, 6): + # Python 2.6: json is part of the standard library + import json + json_decode = json.JSONDecoder().decode + json_encode = json.JSONEncoder().iterencode + else: + json = None + def json_decode(s): + raise ValueError('No JSON library is available') + def json_encode(s): + raise ValueError('No JSON library is available') + +try: + import cPickle as pickle +except ImportError: + # In Python 2, pickle is a Python version. + # In Python 3, pickle is the sped-up C version. + import pickle + +try: + os.urandom(20) + import binascii + def random20(): + return binascii.hexlify(os.urandom(20)).decode('ascii') +except (AttributeError, NotImplementedError): + import random + # os.urandom not available until Python 2.4. Fall back to random.random. + def random20(): + return sha('%s' % random.random()).hexdigest() + +try: + from _thread import get_ident as get_thread_ident +except ImportError: + from thread import get_ident as get_thread_ident + +try: + # Python 3 + next = next +except NameError: + # Python 2 + def next(i): + return i.next() + diff --git a/cherrypy/_cpconfig.py b/cherrypy/_cpconfig.py new file mode 100755 index 0000000..7b4c6a4 --- /dev/null +++ b/cherrypy/_cpconfig.py @@ -0,0 +1,295 @@ +""" +Configuration system for CherryPy. + +Configuration in CherryPy is implemented via dictionaries. Keys are strings +which name the mapped value, which may be of any type. + + +Architecture +------------ + +CherryPy Requests are part of an Application, which runs in a global context, +and configuration data may apply to any of those three scopes: + +Global + Configuration entries which apply everywhere are stored in + cherrypy.config. + +Application + Entries which apply to each mounted application are stored + on the Application object itself, as 'app.config'. This is a two-level + dict where each key is a path, or "relative URL" (for example, "/" or + "/path/to/my/page"), and each value is a config dict. Usually, this + data is provided in the call to tree.mount(root(), config=conf), + although you may also use app.merge(conf). + +Request + Each Request object possesses a single 'Request.config' dict. + Early in the request process, this dict is populated by merging global + config entries, Application entries (whose path equals or is a parent + of Request.path_info), and any config acquired while looking up the + page handler (see next). + + +Declaration +----------- + +Configuration data may be supplied as a Python dictionary, as a filename, +or as an open file object. When you supply a filename or file, CherryPy +uses Python's builtin ConfigParser; you declare Application config by +writing each path as a section header:: + + [/path/to/my/page] + request.stream = True + +To declare global configuration entries, place them in a [global] section. + +You may also declare config entries directly on the classes and methods +(page handlers) that make up your CherryPy application via the ``_cp_config`` +attribute. For example:: + + class Demo: + _cp_config = {'tools.gzip.on': True} + + def index(self): + return "Hello world" + index.exposed = True + index._cp_config = {'request.show_tracebacks': False} + +.. note:: + + This behavior is only guaranteed for the default dispatcher. + Other dispatchers may have different restrictions on where + you can attach _cp_config attributes. + + +Namespaces +---------- + +Configuration keys are separated into namespaces by the first "." in the key. +Current namespaces: + +engine + Controls the 'application engine', including autoreload. + These can only be declared in the global config. + +tree + Grafts cherrypy.Application objects onto cherrypy.tree. + These can only be declared in the global config. + +hooks + Declares additional request-processing functions. + +log + Configures the logging for each application. + These can only be declared in the global or / config. + +request + Adds attributes to each Request. + +response + Adds attributes to each Response. + +server + Controls the default HTTP server via cherrypy.server. + These can only be declared in the global config. + +tools + Runs and configures additional request-processing packages. + +wsgi + Adds WSGI middleware to an Application's "pipeline". + These can only be declared in the app's root config ("/"). + +checker + Controls the 'checker', which looks for common errors in + app state (including config) when the engine starts. + Global config only. + +The only key that does not exist in a namespace is the "environment" entry. +This special entry 'imports' other config entries from a template stored in +cherrypy._cpconfig.environments[environment]. It only applies to the global +config, and only when you use cherrypy.config.update. + +You can define your own namespaces to be called at the Global, Application, +or Request level, by adding a named handler to cherrypy.config.namespaces, +app.namespaces, or app.request_class.namespaces. The name can +be any string, and the handler must be either a callable or a (Python 2.5 +style) context manager. +""" + +import cherrypy +from cherrypy._cpcompat import set, basestring +from cherrypy.lib import reprconf + +# Deprecated in CherryPy 3.2--remove in 3.3 +NamespaceSet = reprconf.NamespaceSet + +def merge(base, other): + """Merge one app config (from a dict, file, or filename) into another. + + If the given config is a filename, it will be appended to + the list of files to monitor for "autoreload" changes. + """ + if isinstance(other, basestring): + cherrypy.engine.autoreload.files.add(other) + + # Load other into base + for section, value_map in reprconf.as_dict(other).items(): + if not isinstance(value_map, dict): + raise ValueError( + "Application config must include section headers, but the " + "config you tried to merge doesn't have any sections. " + "Wrap your config in another dict with paths as section " + "headers, for example: {'/': config}.") + base.setdefault(section, {}).update(value_map) + + +class Config(reprconf.Config): + """The 'global' configuration data for the entire CherryPy process.""" + + def update(self, config): + """Update self from a dict, file or filename.""" + if isinstance(config, basestring): + # Filename + cherrypy.engine.autoreload.files.add(config) + reprconf.Config.update(self, config) + + def _apply(self, config): + """Update self from a dict.""" + if isinstance(config.get("global", None), dict): + if len(config) > 1: + cherrypy.checker.global_config_contained_paths = True + config = config["global"] + if 'tools.staticdir.dir' in config: + config['tools.staticdir.section'] = "global" + reprconf.Config._apply(self, config) + + def __call__(self, *args, **kwargs): + """Decorator for page handlers to set _cp_config.""" + if args: + raise TypeError( + "The cherrypy.config decorator does not accept positional " + "arguments; you must use keyword arguments.") + def tool_decorator(f): + if not hasattr(f, "_cp_config"): + f._cp_config = {} + for k, v in kwargs.items(): + f._cp_config[k] = v + return f + return tool_decorator + + +Config.environments = environments = { + "staging": { + 'engine.autoreload_on': False, + 'checker.on': False, + 'tools.log_headers.on': False, + 'request.show_tracebacks': False, + 'request.show_mismatched_params': False, + }, + "production": { + 'engine.autoreload_on': False, + 'checker.on': False, + 'tools.log_headers.on': False, + 'request.show_tracebacks': False, + 'request.show_mismatched_params': False, + 'log.screen': False, + }, + "embedded": { + # For use with CherryPy embedded in another deployment stack. + 'engine.autoreload_on': False, + 'checker.on': False, + 'tools.log_headers.on': False, + 'request.show_tracebacks': False, + 'request.show_mismatched_params': False, + 'log.screen': False, + 'engine.SIGHUP': None, + 'engine.SIGTERM': None, + }, + "test_suite": { + 'engine.autoreload_on': False, + 'checker.on': False, + 'tools.log_headers.on': False, + 'request.show_tracebacks': True, + 'request.show_mismatched_params': True, + 'log.screen': False, + }, + } + + +def _server_namespace_handler(k, v): + """Config handler for the "server" namespace.""" + atoms = k.split(".", 1) + if len(atoms) > 1: + # Special-case config keys of the form 'server.servername.socket_port' + # to configure additional HTTP servers. + if not hasattr(cherrypy, "servers"): + cherrypy.servers = {} + + servername, k = atoms + if servername not in cherrypy.servers: + from cherrypy import _cpserver + cherrypy.servers[servername] = _cpserver.Server() + # On by default, but 'on = False' can unsubscribe it (see below). + cherrypy.servers[servername].subscribe() + + if k == 'on': + if v: + cherrypy.servers[servername].subscribe() + else: + cherrypy.servers[servername].unsubscribe() + else: + setattr(cherrypy.servers[servername], k, v) + else: + setattr(cherrypy.server, k, v) +Config.namespaces["server"] = _server_namespace_handler + +def _engine_namespace_handler(k, v): + """Backward compatibility handler for the "engine" namespace.""" + engine = cherrypy.engine + if k == 'autoreload_on': + if v: + engine.autoreload.subscribe() + else: + engine.autoreload.unsubscribe() + elif k == 'autoreload_frequency': + engine.autoreload.frequency = v + elif k == 'autoreload_match': + engine.autoreload.match = v + elif k == 'reload_files': + engine.autoreload.files = set(v) + elif k == 'deadlock_poll_freq': + engine.timeout_monitor.frequency = v + elif k == 'SIGHUP': + engine.listeners['SIGHUP'] = set([v]) + elif k == 'SIGTERM': + engine.listeners['SIGTERM'] = set([v]) + elif "." in k: + plugin, attrname = k.split(".", 1) + plugin = getattr(engine, plugin) + if attrname == 'on': + if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): + plugin.subscribe() + return + elif (not v) and hasattr(getattr(plugin, 'unsubscribe', None), '__call__'): + plugin.unsubscribe() + return + setattr(plugin, attrname, v) + else: + setattr(engine, k, v) +Config.namespaces["engine"] = _engine_namespace_handler + + +def _tree_namespace_handler(k, v): + """Namespace handler for the 'tree' config namespace.""" + if isinstance(v, dict): + for script_name, app in v.items(): + cherrypy.tree.graft(app, script_name) + cherrypy.engine.log("Mounted: %s on %s" % (app, script_name or "/")) + else: + cherrypy.tree.graft(v, v.script_name) + cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) +Config.namespaces["tree"] = _tree_namespace_handler + + diff --git a/cherrypy/_cpdispatch.py b/cherrypy/_cpdispatch.py new file mode 100755 index 0000000..7250ac9 --- /dev/null +++ b/cherrypy/_cpdispatch.py @@ -0,0 +1,622 @@ +"""CherryPy dispatchers. + +A 'dispatcher' is the object which looks up the 'page handler' callable +and collects config for the current request based on the path_info, other +request attributes, and the application architecture. The core calls the +dispatcher as early as possible, passing it a 'path_info' argument. + +The default dispatcher discovers the page handler by matching path_info +to a hierarchical arrangement of objects, starting at request.app.root. +""" + +import string +import sys +import types + +import cherrypy + + +class PageHandler(object): + """Callable which sets response.body.""" + + def __init__(self, callable, *args, **kwargs): + self.callable = callable + self.args = args + self.kwargs = kwargs + + def __call__(self): + try: + return self.callable(*self.args, **self.kwargs) + except TypeError: + x = sys.exc_info()[1] + try: + test_callable_spec(self.callable, self.args, self.kwargs) + except cherrypy.HTTPError: + raise sys.exc_info()[1] + except: + raise x + raise + + +def test_callable_spec(callable, callable_args, callable_kwargs): + """ + Inspect callable and test to see if the given args are suitable for it. + + When an error occurs during the handler's invoking stage there are 2 + erroneous cases: + 1. Too many parameters passed to a function which doesn't define + one of *args or **kwargs. + 2. Too little parameters are passed to the function. + + There are 3 sources of parameters to a cherrypy handler. + 1. query string parameters are passed as keyword parameters to the handler. + 2. body parameters are also passed as keyword parameters. + 3. when partial matching occurs, the final path atoms are passed as + positional args. + Both the query string and path atoms are part of the URI. If they are + incorrect, then a 404 Not Found should be raised. Conversely the body + parameters are part of the request; if they are invalid a 400 Bad Request. + """ + show_mismatched_params = getattr( + cherrypy.serving.request, 'show_mismatched_params', False) + try: + (args, varargs, varkw, defaults) = inspect.getargspec(callable) + except TypeError: + if isinstance(callable, object) and hasattr(callable, '__call__'): + (args, varargs, varkw, defaults) = inspect.getargspec(callable.__call__) + else: + # If it wasn't one of our own types, re-raise + # the original error + raise + + if args and args[0] == 'self': + args = args[1:] + + arg_usage = dict([(arg, 0,) for arg in args]) + vararg_usage = 0 + varkw_usage = 0 + extra_kwargs = set() + + for i, value in enumerate(callable_args): + try: + arg_usage[args[i]] += 1 + except IndexError: + vararg_usage += 1 + + for key in callable_kwargs.keys(): + try: + arg_usage[key] += 1 + except KeyError: + varkw_usage += 1 + extra_kwargs.add(key) + + # figure out which args have defaults. + args_with_defaults = args[-len(defaults or []):] + for i, val in enumerate(defaults or []): + # Defaults take effect only when the arg hasn't been used yet. + if arg_usage[args_with_defaults[i]] == 0: + arg_usage[args_with_defaults[i]] += 1 + + missing_args = [] + multiple_args = [] + for key, usage in arg_usage.items(): + if usage == 0: + missing_args.append(key) + elif usage > 1: + multiple_args.append(key) + + if missing_args: + # In the case where the method allows body arguments + # there are 3 potential errors: + # 1. not enough query string parameters -> 404 + # 2. not enough body parameters -> 400 + # 3. not enough path parts (partial matches) -> 404 + # + # We can't actually tell which case it is, + # so I'm raising a 404 because that covers 2/3 of the + # possibilities + # + # In the case where the method does not allow body + # arguments it's definitely a 404. + message = None + if show_mismatched_params: + message="Missing parameters: %s" % ",".join(missing_args) + raise cherrypy.HTTPError(404, message=message) + + # the extra positional arguments come from the path - 404 Not Found + if not varargs and vararg_usage > 0: + raise cherrypy.HTTPError(404) + + body_params = cherrypy.serving.request.body.params or {} + body_params = set(body_params.keys()) + qs_params = set(callable_kwargs.keys()) - body_params + + if multiple_args: + if qs_params.intersection(set(multiple_args)): + # If any of the multiple parameters came from the query string then + # it's a 404 Not Found + error = 404 + else: + # Otherwise it's a 400 Bad Request + error = 400 + + message = None + if show_mismatched_params: + message="Multiple values for parameters: "\ + "%s" % ",".join(multiple_args) + raise cherrypy.HTTPError(error, message=message) + + if not varkw and varkw_usage > 0: + + # If there were extra query string parameters, it's a 404 Not Found + extra_qs_params = set(qs_params).intersection(extra_kwargs) + if extra_qs_params: + message = None + if show_mismatched_params: + message="Unexpected query string "\ + "parameters: %s" % ", ".join(extra_qs_params) + raise cherrypy.HTTPError(404, message=message) + + # If there were any extra body parameters, it's a 400 Not Found + extra_body_params = set(body_params).intersection(extra_kwargs) + if extra_body_params: + message = None + if show_mismatched_params: + message="Unexpected body parameters: "\ + "%s" % ", ".join(extra_body_params) + raise cherrypy.HTTPError(400, message=message) + + +try: + import inspect +except ImportError: + test_callable_spec = lambda callable, args, kwargs: None + + + +class LateParamPageHandler(PageHandler): + """When passing cherrypy.request.params to the page handler, we do not + want to capture that dict too early; we want to give tools like the + decoding tool a chance to modify the params dict in-between the lookup + of the handler and the actual calling of the handler. This subclass + takes that into account, and allows request.params to be 'bound late' + (it's more complicated than that, but that's the effect). + """ + + def _get_kwargs(self): + kwargs = cherrypy.serving.request.params.copy() + if self._kwargs: + kwargs.update(self._kwargs) + return kwargs + + def _set_kwargs(self, kwargs): + self._kwargs = kwargs + + kwargs = property(_get_kwargs, _set_kwargs, + doc='page handler kwargs (with ' + 'cherrypy.request.params copied in)') + + +punctuation_to_underscores = string.maketrans( + string.punctuation, '_' * len(string.punctuation)) + +class Dispatcher(object): + """CherryPy Dispatcher which walks a tree of objects to find a handler. + + The tree is rooted at cherrypy.request.app.root, and each hierarchical + component in the path_info argument is matched to a corresponding nested + attribute of the root object. Matching handlers must have an 'exposed' + attribute which evaluates to True. The special method name "index" + matches a URI which ends in a slash ("/"). The special method name + "default" may match a portion of the path_info (but only when no longer + substring of the path_info matches some other object). + + This is the default, built-in dispatcher for CherryPy. + """ + + dispatch_method_name = '_cp_dispatch' + """ + The name of the dispatch method that nodes may optionally implement + to provide their own dynamic dispatch algorithm. + """ + + def __init__(self, dispatch_method_name=None, + translate=punctuation_to_underscores): + if not isinstance(translate, str) or len(translate) != 256: + raise ValueError("The translate argument must be a str of len 256.") + self.translate = translate + if dispatch_method_name: + self.dispatch_method_name = dispatch_method_name + + def __call__(self, path_info): + """Set handler and config for the current request.""" + request = cherrypy.serving.request + func, vpath = self.find_handler(path_info) + + if func: + # Decode any leftover %2F in the virtual_path atoms. + vpath = [x.replace("%2F", "/") for x in vpath] + request.handler = LateParamPageHandler(func, *vpath) + else: + request.handler = cherrypy.NotFound() + + def find_handler(self, path): + """Return the appropriate page handler, plus any virtual path. + + This will return two objects. The first will be a callable, + which can be used to generate page output. Any parameters from + the query string or request body will be sent to that callable + as keyword arguments. + + The callable is found by traversing the application's tree, + starting from cherrypy.request.app.root, and matching path + components to successive objects in the tree. For example, the + URL "/path/to/handler" might return root.path.to.handler. + + The second object returned will be a list of names which are + 'virtual path' components: parts of the URL which are dynamic, + and were not used when looking up the handler. + These virtual path components are passed to the handler as + positional arguments. + """ + request = cherrypy.serving.request + app = request.app + root = app.root + dispatch_name = self.dispatch_method_name + + # Get config for the root object/path. + fullpath = [x for x in path.strip('/').split('/') if x] + ['index'] + fullpath_len = len(fullpath) + segleft = fullpath_len + nodeconf = {} + if hasattr(root, "_cp_config"): + nodeconf.update(root._cp_config) + if "/" in app.config: + nodeconf.update(app.config["/"]) + object_trail = [['root', root, nodeconf, segleft]] + + node = root + iternames = fullpath[:] + while iternames: + name = iternames[0] + # map to legal Python identifiers (e.g. replace '.' with '_') + objname = name.translate(self.translate) + + nodeconf = {} + subnode = getattr(node, objname, None) + pre_len = len(iternames) + if subnode is None: + dispatch = getattr(node, dispatch_name, None) + if dispatch and hasattr(dispatch, '__call__') and not \ + getattr(dispatch, 'exposed', False) and \ + pre_len > 1: + #Don't expose the hidden 'index' token to _cp_dispatch + #We skip this if pre_len == 1 since it makes no sense + #to call a dispatcher when we have no tokens left. + index_name = iternames.pop() + subnode = dispatch(vpath=iternames) + iternames.append(index_name) + else: + #We didn't find a path, but keep processing in case there + #is a default() handler. + iternames.pop(0) + else: + #We found the path, remove the vpath entry + iternames.pop(0) + segleft = len(iternames) + if segleft > pre_len: + #No path segment was removed. Raise an error. + raise cherrypy.CherryPyException( + "A vpath segment was added. Custom dispatchers may only " + + "remove elements. While trying to process " + + "{0} in {1}".format(name, fullpath) + ) + elif segleft == pre_len: + #Assume that the handler used the current path segment, but + #did not pop it. This allows things like + #return getattr(self, vpath[0], None) + iternames.pop(0) + segleft -= 1 + node = subnode + + if node is not None: + # Get _cp_config attached to this node. + if hasattr(node, "_cp_config"): + nodeconf.update(node._cp_config) + + # Mix in values from app.config for this path. + existing_len = fullpath_len - pre_len + if existing_len != 0: + curpath = '/' + '/'.join(fullpath[0:existing_len]) + else: + curpath = '' + new_segs = fullpath[fullpath_len - pre_len:fullpath_len - segleft] + for seg in new_segs: + curpath += '/' + seg + if curpath in app.config: + nodeconf.update(app.config[curpath]) + + object_trail.append([name, node, nodeconf, segleft]) + + def set_conf(): + """Collapse all object_trail config into cherrypy.request.config.""" + base = cherrypy.config.copy() + # Note that we merge the config from each node + # even if that node was None. + for name, obj, conf, segleft in object_trail: + base.update(conf) + if 'tools.staticdir.dir' in conf: + base['tools.staticdir.section'] = '/' + '/'.join(fullpath[0:fullpath_len - segleft]) + return base + + # Try successive objects (reverse order) + num_candidates = len(object_trail) - 1 + for i in range(num_candidates, -1, -1): + + name, candidate, nodeconf, segleft = object_trail[i] + if candidate is None: + continue + + # Try a "default" method on the current leaf. + if hasattr(candidate, "default"): + defhandler = candidate.default + if getattr(defhandler, 'exposed', False): + # Insert any extra _cp_config from the default handler. + conf = getattr(defhandler, "_cp_config", {}) + object_trail.insert(i+1, ["default", defhandler, conf, segleft]) + request.config = set_conf() + # See http://www.cherrypy.org/ticket/613 + request.is_index = path.endswith("/") + return defhandler, fullpath[fullpath_len - segleft:-1] + + # Uncomment the next line to restrict positional params to "default". + # if i < num_candidates - 2: continue + + # Try the current leaf. + if getattr(candidate, 'exposed', False): + request.config = set_conf() + if i == num_candidates: + # We found the extra ".index". Mark request so tools + # can redirect if path_info has no trailing slash. + request.is_index = True + else: + # We're not at an 'index' handler. Mark request so tools + # can redirect if path_info has NO trailing slash. + # Note that this also includes handlers which take + # positional parameters (virtual paths). + request.is_index = False + return candidate, fullpath[fullpath_len - segleft:-1] + + # We didn't find anything + request.config = set_conf() + return None, [] + + +class MethodDispatcher(Dispatcher): + """Additional dispatch based on cherrypy.request.method.upper(). + + Methods named GET, POST, etc will be called on an exposed class. + The method names must be all caps; the appropriate Allow header + will be output showing all capitalized method names as allowable + HTTP verbs. + + Note that the containing class must be exposed, not the methods. + """ + + def __call__(self, path_info): + """Set handler and config for the current request.""" + request = cherrypy.serving.request + resource, vpath = self.find_handler(path_info) + + if resource: + # Set Allow header + avail = [m for m in dir(resource) if m.isupper()] + if "GET" in avail and "HEAD" not in avail: + avail.append("HEAD") + avail.sort() + cherrypy.serving.response.headers['Allow'] = ", ".join(avail) + + # Find the subhandler + meth = request.method.upper() + func = getattr(resource, meth, None) + if func is None and meth == "HEAD": + func = getattr(resource, "GET", None) + if func: + # Grab any _cp_config on the subhandler. + if hasattr(func, "_cp_config"): + request.config.update(func._cp_config) + + # Decode any leftover %2F in the virtual_path atoms. + vpath = [x.replace("%2F", "/") for x in vpath] + request.handler = LateParamPageHandler(func, *vpath) + else: + request.handler = cherrypy.HTTPError(405) + else: + request.handler = cherrypy.NotFound() + + +class RoutesDispatcher(object): + """A Routes based dispatcher for CherryPy.""" + + def __init__(self, full_result=False): + """ + Routes dispatcher + + Set full_result to True if you wish the controller + and the action to be passed on to the page handler + parameters. By default they won't be. + """ + import routes + self.full_result = full_result + self.controllers = {} + self.mapper = routes.Mapper() + self.mapper.controller_scan = self.controllers.keys + + def connect(self, name, route, controller, **kwargs): + self.controllers[name] = controller + self.mapper.connect(name, route, controller=name, **kwargs) + + def redirect(self, url): + raise cherrypy.HTTPRedirect(url) + + def __call__(self, path_info): + """Set handler and config for the current request.""" + func = self.find_handler(path_info) + if func: + cherrypy.serving.request.handler = LateParamPageHandler(func) + else: + cherrypy.serving.request.handler = cherrypy.NotFound() + + def find_handler(self, path_info): + """Find the right page handler, and set request.config.""" + import routes + + request = cherrypy.serving.request + + config = routes.request_config() + config.mapper = self.mapper + if hasattr(request, 'wsgi_environ'): + config.environ = request.wsgi_environ + config.host = request.headers.get('Host', None) + config.protocol = request.scheme + config.redirect = self.redirect + + result = self.mapper.match(path_info) + + config.mapper_dict = result + params = {} + if result: + params = result.copy() + if not self.full_result: + params.pop('controller', None) + params.pop('action', None) + request.params.update(params) + + # Get config for the root object/path. + request.config = base = cherrypy.config.copy() + curpath = "" + + def merge(nodeconf): + if 'tools.staticdir.dir' in nodeconf: + nodeconf['tools.staticdir.section'] = curpath or "/" + base.update(nodeconf) + + app = request.app + root = app.root + if hasattr(root, "_cp_config"): + merge(root._cp_config) + if "/" in app.config: + merge(app.config["/"]) + + # Mix in values from app.config. + atoms = [x for x in path_info.split("/") if x] + if atoms: + last = atoms.pop() + else: + last = None + for atom in atoms: + curpath = "/".join((curpath, atom)) + if curpath in app.config: + merge(app.config[curpath]) + + handler = None + if result: + controller = result.get('controller') + controller = self.controllers.get(controller, controller) + if controller: + if isinstance(controller, (type, types.ClassType)): + controller = controller() + # Get config from the controller. + if hasattr(controller, "_cp_config"): + merge(controller._cp_config) + + action = result.get('action') + if action is not None: + handler = getattr(controller, action, None) + # Get config from the handler + if hasattr(handler, "_cp_config"): + merge(handler._cp_config) + else: + handler = controller + + # Do the last path atom here so it can + # override the controller's _cp_config. + if last: + curpath = "/".join((curpath, last)) + if curpath in app.config: + merge(app.config[curpath]) + + return handler + + +def XMLRPCDispatcher(next_dispatcher=Dispatcher()): + from cherrypy.lib import xmlrpc + def xmlrpc_dispatch(path_info): + path_info = xmlrpc.patched_path(path_info) + return next_dispatcher(path_info) + return xmlrpc_dispatch + + +def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, **domains): + """ + Select a different handler based on the Host header. + + This can be useful when running multiple sites within one CP server. + It allows several domains to point to different parts of a single + website structure. For example:: + + http://www.domain.example -> root + http://www.domain2.example -> root/domain2/ + http://www.domain2.example:443 -> root/secure + + can be accomplished via the following config:: + + [/] + request.dispatch = cherrypy.dispatch.VirtualHost( + **{'www.domain2.example': '/domain2', + 'www.domain2.example:443': '/secure', + }) + + next_dispatcher + The next dispatcher object in the dispatch chain. + The VirtualHost dispatcher adds a prefix to the URL and calls + another dispatcher. Defaults to cherrypy.dispatch.Dispatcher(). + + use_x_forwarded_host + If True (the default), any "X-Forwarded-Host" + request header will be used instead of the "Host" header. This + is commonly added by HTTP servers (such as Apache) when proxying. + + ``**domains`` + A dict of {host header value: virtual prefix} pairs. + The incoming "Host" request header is looked up in this dict, + and, if a match is found, the corresponding "virtual prefix" + value will be prepended to the URL path before calling the + next dispatcher. Note that you often need separate entries + for "example.com" and "www.example.com". In addition, "Host" + headers may contain the port number. + """ + from cherrypy.lib import httputil + def vhost_dispatch(path_info): + request = cherrypy.serving.request + header = request.headers.get + + domain = header('Host', '') + if use_x_forwarded_host: + domain = header("X-Forwarded-Host", domain) + + prefix = domains.get(domain, "") + if prefix: + path_info = httputil.urljoin(prefix, path_info) + + result = next_dispatcher(path_info) + + # Touch up staticdir config. See http://www.cherrypy.org/ticket/614. + section = request.config.get('tools.staticdir.section') + if section: + section = section[len(prefix):] + request.config['tools.staticdir.section'] = section + + return result + return vhost_dispatch + diff --git a/cherrypy/_cperror.py b/cherrypy/_cperror.py new file mode 100755 index 0000000..00e5b53 --- /dev/null +++ b/cherrypy/_cperror.py @@ -0,0 +1,553 @@ +"""Exception classes for CherryPy. + +CherryPy provides (and uses) exceptions for declaring that the HTTP response +should be a status other than the default "200 OK". You can ``raise`` them like +normal Python exceptions. You can also call them and they will raise themselves; +this means you can set an :class:`HTTPError<cherrypy._cperror.HTTPError>` +or :class:`HTTPRedirect<cherrypy._cperror.HTTPRedirect>` as the +:attr:`request.handler<cherrypy._cprequest.Request.handler>`. + +.. _redirectingpost: + +Redirecting POST +================ + +When you GET a resource and are redirected by the server to another Location, +there's generally no problem since GET is both a "safe method" (there should +be no side-effects) and an "idempotent method" (multiple calls are no different +than a single call). + +POST, however, is neither safe nor idempotent--if you +charge a credit card, you don't want to be charged twice by a redirect! + +For this reason, *none* of the 3xx responses permit a user-agent (browser) to +resubmit a POST on redirection without first confirming the action with the user: + +===== ================================= =========== +300 Multiple Choices Confirm with the user +301 Moved Permanently Confirm with the user +302 Found (Object moved temporarily) Confirm with the user +303 See Other GET the new URI--no confirmation +304 Not modified (for conditional GET only--POST should not raise this error) +305 Use Proxy Confirm with the user +307 Temporary Redirect Confirm with the user +===== ================================= =========== + +However, browsers have historically implemented these restrictions poorly; +in particular, many browsers do not force the user to confirm 301, 302 +or 307 when redirecting POST. For this reason, CherryPy defaults to 303, +which most user-agents appear to have implemented correctly. Therefore, if +you raise HTTPRedirect for a POST request, the user-agent will most likely +attempt to GET the new URI (without asking for confirmation from the user). +We realize this is confusing for developers, but it's the safest thing we +could do. You are of course free to raise ``HTTPRedirect(uri, status=302)`` +or any other 3xx status if you know what you're doing, but given the +environment, we couldn't let any of those be the default. + +Custom Error Handling +===================== + +.. image:: /refman/cperrors.gif + +Anticipated HTTP responses +-------------------------- + +The 'error_page' config namespace can be used to provide custom HTML output for +expected responses (like 404 Not Found). Supply a filename from which the output +will be read. The contents will be interpolated with the values %(status)s, +%(message)s, %(traceback)s, and %(version)s using plain old Python +`string formatting <http://www.python.org/doc/2.6.4/library/stdtypes.html#string-formatting-operations>`_. + +:: + + _cp_config = {'error_page.404': os.path.join(localDir, "static/index.html")} + + +Beginning in version 3.1, you may also provide a function or other callable as +an error_page entry. It will be passed the same status, message, traceback and +version arguments that are interpolated into templates:: + + def error_page_402(status, message, traceback, version): + return "Error %s - Well, I'm very sorry but you haven't paid!" % status + cherrypy.config.update({'error_page.402': error_page_402}) + +Also in 3.1, in addition to the numbered error codes, you may also supply +"error_page.default" to handle all codes which do not have their own error_page entry. + + + +Unanticipated errors +-------------------- + +CherryPy also has a generic error handling mechanism: whenever an unanticipated +error occurs in your code, it will call +:func:`Request.error_response<cherrypy._cprequest.Request.error_response>` to set +the response status, headers, and body. By default, this is the same output as +:class:`HTTPError(500) <cherrypy._cperror.HTTPError>`. If you want to provide +some other behavior, you generally replace "request.error_response". + +Here is some sample code that shows how to display a custom error message and +send an e-mail containing the error:: + + from cherrypy import _cperror + + def handle_error(): + cherrypy.response.status = 500 + cherrypy.response.body = ["<html><body>Sorry, an error occured</body></html>"] + sendMail('error@domain.com', 'Error in your web app', _cperror.format_exc()) + + class Root: + _cp_config = {'request.error_response': handle_error} + + +Note that you have to explicitly set :attr:`response.body <cherrypy._cprequest.Response.body>` +and not simply return an error message as a result. +""" + +from cgi import escape as _escape +from sys import exc_info as _exc_info +from traceback import format_exception as _format_exception +from cherrypy._cpcompat import basestring, iteritems, urljoin as _urljoin +from cherrypy.lib import httputil as _httputil + + +class CherryPyException(Exception): + """A base class for CherryPy exceptions.""" + pass + + +class TimeoutError(CherryPyException): + """Exception raised when Response.timed_out is detected.""" + pass + + +class InternalRedirect(CherryPyException): + """Exception raised to switch to the handler for a different URL. + + This exception will redirect processing to another path within the site + (without informing the client). Provide the new path as an argument when + raising the exception. Provide any params in the querystring for the new URL. + """ + + def __init__(self, path, query_string=""): + import cherrypy + self.request = cherrypy.serving.request + + self.query_string = query_string + if "?" in path: + # Separate any params included in the path + path, self.query_string = path.split("?", 1) + + # Note that urljoin will "do the right thing" whether url is: + # 1. a URL relative to root (e.g. "/dummy") + # 2. a URL relative to the current path + # Note that any query string will be discarded. + path = _urljoin(self.request.path_info, path) + + # Set a 'path' member attribute so that code which traps this + # error can have access to it. + self.path = path + + CherryPyException.__init__(self, path, self.query_string) + + +class HTTPRedirect(CherryPyException): + """Exception raised when the request should be redirected. + + This exception will force a HTTP redirect to the URL or URL's you give it. + The new URL must be passed as the first argument to the Exception, + e.g., HTTPRedirect(newUrl). Multiple URLs are allowed in a list. + If a URL is absolute, it will be used as-is. If it is relative, it is + assumed to be relative to the current cherrypy.request.path_info. + + If one of the provided URL is a unicode object, it will be encoded + using the default encoding or the one passed in parameter. + + There are multiple types of redirect, from which you can select via the + ``status`` argument. If you do not provide a ``status`` arg, it defaults to + 303 (or 302 if responding with HTTP/1.0). + + Examples:: + + raise cherrypy.HTTPRedirect("") + raise cherrypy.HTTPRedirect("/abs/path", 307) + raise cherrypy.HTTPRedirect(["path1", "path2?a=1&b=2"], 301) + + See :ref:`redirectingpost` for additional caveats. + """ + + status = None + """The integer HTTP status code to emit.""" + + urls = None + """The list of URL's to emit.""" + + encoding = 'utf-8' + """The encoding when passed urls are unicode objects""" + + def __init__(self, urls, status=None, encoding=None): + import cherrypy + request = cherrypy.serving.request + + if isinstance(urls, basestring): + urls = [urls] + + abs_urls = [] + for url in urls: + if isinstance(url, unicode): + url = url.encode(encoding or self.encoding) + + # Note that urljoin will "do the right thing" whether url is: + # 1. a complete URL with host (e.g. "http://www.example.com/test") + # 2. a URL relative to root (e.g. "/dummy") + # 3. a URL relative to the current path + # Note that any query string in cherrypy.request is discarded. + url = _urljoin(cherrypy.url(), url) + abs_urls.append(url) + self.urls = abs_urls + + # RFC 2616 indicates a 301 response code fits our goal; however, + # browser support for 301 is quite messy. Do 302/303 instead. See + # http://www.alanflavell.org.uk/www/post-redirect.html + if status is None: + if request.protocol >= (1, 1): + status = 303 + else: + status = 302 + else: + status = int(status) + if status < 300 or status > 399: + raise ValueError("status must be between 300 and 399.") + + self.status = status + CherryPyException.__init__(self, abs_urls, status) + + def set_response(self): + """Modify cherrypy.response status, headers, and body to represent self. + + CherryPy uses this internally, but you can also use it to create an + HTTPRedirect object and set its output without *raising* the exception. + """ + import cherrypy + response = cherrypy.serving.response + response.status = status = self.status + + if status in (300, 301, 302, 303, 307): + response.headers['Content-Type'] = "text/html;charset=utf-8" + # "The ... URI SHOULD be given by the Location field + # in the response." + response.headers['Location'] = self.urls[0] + + # "Unless the request method was HEAD, the entity of the response + # SHOULD contain a short hypertext note with a hyperlink to the + # new URI(s)." + msg = {300: "This resource can be found at <a href='%s'>%s</a>.", + 301: "This resource has permanently moved to <a href='%s'>%s</a>.", + 302: "This resource resides temporarily at <a href='%s'>%s</a>.", + 303: "This resource can be found at <a href='%s'>%s</a>.", + 307: "This resource has moved temporarily to <a href='%s'>%s</a>.", + }[status] + msgs = [msg % (u, u) for u in self.urls] + response.body = "<br />\n".join(msgs) + # Previous code may have set C-L, so we have to reset it + # (allow finalize to set it). + response.headers.pop('Content-Length', None) + elif status == 304: + # Not Modified. + # "The response MUST include the following header fields: + # Date, unless its omission is required by section 14.18.1" + # The "Date" header should have been set in Response.__init__ + + # "...the response SHOULD NOT include other entity-headers." + for key in ('Allow', 'Content-Encoding', 'Content-Language', + 'Content-Length', 'Content-Location', 'Content-MD5', + 'Content-Range', 'Content-Type', 'Expires', + 'Last-Modified'): + if key in response.headers: + del response.headers[key] + + # "The 304 response MUST NOT contain a message-body." + response.body = None + # Previous code may have set C-L, so we have to reset it. + response.headers.pop('Content-Length', None) + elif status == 305: + # Use Proxy. + # self.urls[0] should be the URI of the proxy. + response.headers['Location'] = self.urls[0] + response.body = None + # Previous code may have set C-L, so we have to reset it. + response.headers.pop('Content-Length', None) + else: + raise ValueError("The %s status code is unknown." % status) + + def __call__(self): + """Use this exception as a request.handler (raise self).""" + raise self + + +def clean_headers(status): + """Remove any headers which should not apply to an error response.""" + import cherrypy + + response = cherrypy.serving.response + + # Remove headers which applied to the original content, + # but do not apply to the error page. + respheaders = response.headers + for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After", + "Vary", "Content-Encoding", "Content-Length", "Expires", + "Content-Location", "Content-MD5", "Last-Modified"]: + if key in respheaders: + del respheaders[key] + + if status != 416: + # A server sending a response with status code 416 (Requested + # range not satisfiable) SHOULD include a Content-Range field + # with a byte-range-resp-spec of "*". The instance-length + # specifies the current length of the selected resource. + # A response with status code 206 (Partial Content) MUST NOT + # include a Content-Range field with a byte-range- resp-spec of "*". + if "Content-Range" in respheaders: + del respheaders["Content-Range"] + + +class HTTPError(CherryPyException): + """Exception used to return an HTTP error code (4xx-5xx) to the client. + + This exception can be used to automatically send a response using a http status + code, with an appropriate error page. It takes an optional + ``status`` argument (which must be between 400 and 599); it defaults to 500 + ("Internal Server Error"). It also takes an optional ``message`` argument, + which will be returned in the response body. See + `RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4>`_ + for a complete list of available error codes and when to use them. + + Examples:: + + raise cherrypy.HTTPError(403) + raise cherrypy.HTTPError("403 Forbidden", "You are not allowed to access this resource.") + """ + + status = None + """The HTTP status code. May be of type int or str (with a Reason-Phrase).""" + + code = None + """The integer HTTP status code.""" + + reason = None + """The HTTP Reason-Phrase string.""" + + def __init__(self, status=500, message=None): + self.status = status + try: + self.code, self.reason, defaultmsg = _httputil.valid_status(status) + except ValueError, x: + raise self.__class__(500, x.args[0]) + + if self.code < 400 or self.code > 599: + raise ValueError("status must be between 400 and 599.") + + # See http://www.python.org/dev/peps/pep-0352/ + # self.message = message + self._message = message or defaultmsg + CherryPyException.__init__(self, status, message) + + def set_response(self): + """Modify cherrypy.response status, headers, and body to represent self. + + CherryPy uses this internally, but you can also use it to create an + HTTPError object and set its output without *raising* the exception. + """ + import cherrypy + + response = cherrypy.serving.response + + clean_headers(self.code) + + # In all cases, finalize will be called after this method, + # so don't bother cleaning up response values here. + response.status = self.status + tb = None + if cherrypy.serving.request.show_tracebacks: + tb = format_exc() + response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers.pop('Content-Length', None) + + content = self.get_error_page(self.status, traceback=tb, + message=self._message) + response.body = content + + _be_ie_unfriendly(self.code) + + def get_error_page(self, *args, **kwargs): + return get_error_page(*args, **kwargs) + + def __call__(self): + """Use this exception as a request.handler (raise self).""" + raise self + + +class NotFound(HTTPError): + """Exception raised when a URL could not be mapped to any handler (404). + + This is equivalent to raising + :class:`HTTPError("404 Not Found") <cherrypy._cperror.HTTPError>`. + """ + + def __init__(self, path=None): + if path is None: + import cherrypy + request = cherrypy.serving.request + path = request.script_name + request.path_info + self.args = (path,) + HTTPError.__init__(self, 404, "The path '%s' was not found." % path) + + +_HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta> + <title>%(status)s</title> + <style type="text/css"> + #powered_by { + margin-top: 20px; + border-top: 2px solid black; + font-style: italic; + } + + #traceback { + color: red; + } + </style> +</head> + <body> + <h2>%(status)s</h2> + <p>%(message)s</p> + <pre id="traceback">%(traceback)s</pre> + <div id="powered_by"> + <span>Powered by <a href="http://www.cherrypy.org">CherryPy %(version)s</a></span> + </div> + </body> +</html> +''' + +def get_error_page(status, **kwargs): + """Return an HTML page, containing a pretty error response. + + status should be an int or a str. + kwargs will be interpolated into the page template. + """ + import cherrypy + + try: + code, reason, message = _httputil.valid_status(status) + except ValueError, x: + raise cherrypy.HTTPError(500, x.args[0]) + + # We can't use setdefault here, because some + # callers send None for kwarg values. + if kwargs.get('status') is None: + kwargs['status'] = "%s %s" % (code, reason) + if kwargs.get('message') is None: + kwargs['message'] = message + if kwargs.get('traceback') is None: + kwargs['traceback'] = '' + if kwargs.get('version') is None: + kwargs['version'] = cherrypy.__version__ + + for k, v in iteritems(kwargs): + if v is None: + kwargs[k] = "" + else: + kwargs[k] = _escape(kwargs[k]) + + # Use a custom template or callable for the error page? + pages = cherrypy.serving.request.error_page + error_page = pages.get(code) or pages.get('default') + if error_page: + try: + if hasattr(error_page, '__call__'): + return error_page(**kwargs) + else: + return open(error_page, 'rb').read() % kwargs + except: + e = _format_exception(*_exc_info())[-1] + m = kwargs['message'] + if m: + m += "<br />" + m += "In addition, the custom error page failed:\n<br />%s" % e + kwargs['message'] = m + + return _HTTPErrorTemplate % kwargs + + +_ie_friendly_error_sizes = { + 400: 512, 403: 256, 404: 512, 405: 256, + 406: 512, 408: 512, 409: 512, 410: 256, + 500: 512, 501: 512, 505: 512, + } + + +def _be_ie_unfriendly(status): + import cherrypy + response = cherrypy.serving.response + + # For some statuses, Internet Explorer 5+ shows "friendly error + # messages" instead of our response.body if the body is smaller + # than a given size. Fix this by returning a body over that size + # (by adding whitespace). + # See http://support.microsoft.com/kb/q218155/ + s = _ie_friendly_error_sizes.get(status, 0) + if s: + s += 1 + # Since we are issuing an HTTP error status, we assume that + # the entity is short, and we should just collapse it. + content = response.collapse_body() + l = len(content) + if l and l < s: + # IN ADDITION: the response must be written to IE + # in one chunk or it will still get replaced! Bah. + content = content + (" " * (s - l)) + response.body = content + response.headers['Content-Length'] = str(len(content)) + + +def format_exc(exc=None): + """Return exc (or sys.exc_info if None), formatted.""" + if exc is None: + exc = _exc_info() + if exc == (None, None, None): + return "" + import traceback + return "".join(traceback.format_exception(*exc)) + +def bare_error(extrabody=None): + """Produce status, headers, body for a critical error. + + Returns a triple without calling any other questionable functions, + so it should be as error-free as possible. Call it from an HTTP server + if you get errors outside of the request. + + If extrabody is None, a friendly but rather unhelpful error message + is set in the body. If extrabody is a string, it will be appended + as-is to the body. + """ + + # The whole point of this function is to be a last line-of-defense + # in handling errors. That is, it must not raise any errors itself; + # it cannot be allowed to fail. Therefore, don't add to it! + # In particular, don't call any other CP functions. + + body = "Unrecoverable error in the server." + if extrabody is not None: + if not isinstance(extrabody, str): + extrabody = extrabody.encode('utf-8') + body += "\n" + extrabody + + return ("500 Internal Server Error", + [('Content-Type', 'text/plain'), + ('Content-Length', str(len(body)))], + [body]) + + diff --git a/cherrypy/_cplogging.py b/cherrypy/_cplogging.py new file mode 100755 index 0000000..d6ca979 --- /dev/null +++ b/cherrypy/_cplogging.py @@ -0,0 +1,393 @@ +""" +Simple config +============= + +Although CherryPy uses the :mod:`Python logging module <logging>`, it does so +behind the scenes so that simple logging is simple, but complicated logging +is still possible. "Simple" logging means that you can log to the screen +(i.e. console/stdout) or to a file, and that you can easily have separate +error and access log files. + +Here are the simplified logging settings. You use these by adding lines to +your config file or dict. You should set these at either the global level or +per application (see next), but generally not both. + + * ``log.screen``: Set this to True to have both "error" and "access" messages + printed to stdout. + * ``log.access_file``: Set this to an absolute filename where you want + "access" messages written. + * ``log.error_file``: Set this to an absolute filename where you want "error" + messages written. + +Many events are automatically logged; to log your own application events, call +:func:`cherrypy.log`. + +Architecture +============ + +Separate scopes +--------------- + +CherryPy provides log managers at both the global and application layers. +This means you can have one set of logging rules for your entire site, +and another set of rules specific to each application. The global log +manager is found at :func:`cherrypy.log`, and the log manager for each +application is found at :attr:`app.log<cherrypy._cptree.Application.log>`. +If you're inside a request, the latter is reachable from +``cherrypy.request.app.log``; if you're outside a request, you'll have to obtain +a reference to the ``app``: either the return value of +:func:`tree.mount()<cherrypy._cptree.Tree.mount>` or, if you used +:func:`quickstart()<cherrypy.quickstart>` instead, via ``cherrypy.tree.apps['/']``. + +By default, the global logs are named "cherrypy.error" and "cherrypy.access", +and the application logs are named "cherrypy.error.2378745" and +"cherrypy.access.2378745" (the number is the id of the Application object). +This means that the application logs "bubble up" to the site logs, so if your +application has no log handlers, the site-level handlers will still log the +messages. + +Errors vs. Access +----------------- + +Each log manager handles both "access" messages (one per HTTP request) and +"error" messages (everything else). Note that the "error" log is not just for +errors! The format of access messages is highly formalized, but the error log +isn't--it receives messages from a variety of sources (including full error +tracebacks, if enabled). + + +Custom Handlers +=============== + +The simple settings above work by manipulating Python's standard :mod:`logging` +module. So when you need something more complex, the full power of the standard +module is yours to exploit. You can borrow or create custom handlers, formats, +filters, and much more. Here's an example that skips the standard FileHandler +and uses a RotatingFileHandler instead: + +:: + + #python + log = app.log + + # Remove the default FileHandlers if present. + log.error_file = "" + log.access_file = "" + + maxBytes = getattr(log, "rot_maxBytes", 10000000) + backupCount = getattr(log, "rot_backupCount", 1000) + + # Make a new RotatingFileHandler for the error log. + fname = getattr(log, "rot_error_file", "error.log") + h = handlers.RotatingFileHandler(fname, 'a', maxBytes, backupCount) + h.setLevel(DEBUG) + h.setFormatter(_cplogging.logfmt) + log.error_log.addHandler(h) + + # Make a new RotatingFileHandler for the access log. + fname = getattr(log, "rot_access_file", "access.log") + h = handlers.RotatingFileHandler(fname, 'a', maxBytes, backupCount) + h.setLevel(DEBUG) + h.setFormatter(_cplogging.logfmt) + log.access_log.addHandler(h) + + +The ``rot_*`` attributes are pulled straight from the application log object. +Since "log.*" config entries simply set attributes on the log object, you can +add custom attributes to your heart's content. Note that these handlers are +used ''instead'' of the default, simple handlers outlined above (so don't set +the "log.error_file" config entry, for example). +""" + +import datetime +import logging +# Silence the no-handlers "warning" (stderr write!) in stdlib logging +logging.Logger.manager.emittedNoHandlerWarning = 1 +logfmt = logging.Formatter("%(message)s") +import os +import sys + +import cherrypy +from cherrypy import _cperror + + +class LogManager(object): + """An object to assist both simple and advanced logging. + + ``cherrypy.log`` is an instance of this class. + """ + + appid = None + """The id() of the Application object which owns this log manager. If this + is a global log manager, appid is None.""" + + error_log = None + """The actual :class:`logging.Logger` instance for error messages.""" + + access_log = None + """The actual :class:`logging.Logger` instance for access messages.""" + + access_log_format = \ + '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + + logger_root = None + """The "top-level" logger name. + + This string will be used as the first segment in the Logger names. + The default is "cherrypy", for example, in which case the Logger names + will be of the form:: + + cherrypy.error.<appid> + cherrypy.access.<appid> + """ + + def __init__(self, appid=None, logger_root="cherrypy"): + self.logger_root = logger_root + self.appid = appid + if appid is None: + self.error_log = logging.getLogger("%s.error" % logger_root) + self.access_log = logging.getLogger("%s.access" % logger_root) + else: + self.error_log = logging.getLogger("%s.error.%s" % (logger_root, appid)) + self.access_log = logging.getLogger("%s.access.%s" % (logger_root, appid)) + self.error_log.setLevel(logging.INFO) + self.access_log.setLevel(logging.INFO) + cherrypy.engine.subscribe('graceful', self.reopen_files) + + def reopen_files(self): + """Close and reopen all file handlers.""" + for log in (self.error_log, self.access_log): + for h in log.handlers: + if isinstance(h, logging.FileHandler): + h.acquire() + h.stream.close() + h.stream = open(h.baseFilename, h.mode) + h.release() + + def error(self, msg='', context='', severity=logging.INFO, traceback=False): + """Write the given ``msg`` to the error log. + + This is not just for errors! Applications may call this at any time + to log application-specific information. + + If ``traceback`` is True, the traceback of the current exception + (if any) will be appended to ``msg``. + """ + if traceback: + msg += _cperror.format_exc() + self.error_log.log(severity, ' '.join((self.time(), context, msg))) + + def __call__(self, *args, **kwargs): + """An alias for ``error``.""" + return self.error(*args, **kwargs) + + def access(self): + """Write to the access log (in Apache/NCSA Combined Log format). + + See http://httpd.apache.org/docs/2.0/logs.html#combined for format + details. + + CherryPy calls this automatically for you. Note there are no arguments; + it collects the data itself from + :class:`cherrypy.request<cherrypy._cprequest.Request>`. + + Like Apache started doing in 2.0.46, non-printable and other special + characters in %r (and we expand that to all parts) are escaped using + \\xhh sequences, where hh stands for the hexadecimal representation + of the raw byte. Exceptions from this rule are " and \\, which are + escaped by prepending a backslash, and all whitespace characters, + which are written in their C-style notation (\\n, \\t, etc). + """ + request = cherrypy.serving.request + remote = request.remote + response = cherrypy.serving.response + outheaders = response.headers + inheaders = request.headers + if response.output_status is None: + status = "-" + else: + status = response.output_status.split(" ", 1)[0] + + atoms = {'h': remote.name or remote.ip, + 'l': '-', + 'u': getattr(request, "login", None) or "-", + 't': self.time(), + 'r': request.request_line, + 's': status, + 'b': dict.get(outheaders, 'Content-Length', '') or "-", + 'f': dict.get(inheaders, 'Referer', ''), + 'a': dict.get(inheaders, 'User-Agent', ''), + } + for k, v in atoms.items(): + if isinstance(v, unicode): + v = v.encode('utf8') + elif not isinstance(v, str): + v = str(v) + # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc + # and backslash for us. All we have to do is strip the quotes. + v = repr(v)[1:-1] + # Escape double-quote. + atoms[k] = v.replace('"', '\\"') + + try: + self.access_log.log(logging.INFO, self.access_log_format % atoms) + except: + self(traceback=True) + + def time(self): + """Return now() in Apache Common Log Format (no timezone).""" + now = datetime.datetime.now() + monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', + 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] + month = monthnames[now.month - 1].capitalize() + return ('[%02d/%s/%04d:%02d:%02d:%02d]' % + (now.day, month, now.year, now.hour, now.minute, now.second)) + + def _get_builtin_handler(self, log, key): + for h in log.handlers: + if getattr(h, "_cpbuiltin", None) == key: + return h + + + # ------------------------- Screen handlers ------------------------- # + + def _set_screen_handler(self, log, enable, stream=None): + h = self._get_builtin_handler(log, "screen") + if enable: + if not h: + if stream is None: + stream=sys.stderr + h = logging.StreamHandler(stream) + h.setFormatter(logfmt) + h._cpbuiltin = "screen" + log.addHandler(h) + elif h: + log.handlers.remove(h) + + def _get_screen(self): + h = self._get_builtin_handler + has_h = h(self.error_log, "screen") or h(self.access_log, "screen") + return bool(has_h) + + def _set_screen(self, newvalue): + self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) + self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) + screen = property(_get_screen, _set_screen, + doc="""Turn stderr/stdout logging on or off. + + If you set this to True, it'll add the appropriate StreamHandler for + you. If you set it to False, it will remove the handler. + """) + + # -------------------------- File handlers -------------------------- # + + def _add_builtin_file_handler(self, log, fname): + h = logging.FileHandler(fname) + h.setFormatter(logfmt) + h._cpbuiltin = "file" + log.addHandler(h) + + def _set_file_handler(self, log, filename): + h = self._get_builtin_handler(log, "file") + if filename: + if h: + if h.baseFilename != os.path.abspath(filename): + h.close() + log.handlers.remove(h) + self._add_builtin_file_handler(log, filename) + else: + self._add_builtin_file_handler(log, filename) + else: + if h: + h.close() + log.handlers.remove(h) + + def _get_error_file(self): + h = self._get_builtin_handler(self.error_log, "file") + if h: + return h.baseFilename + return '' + def _set_error_file(self, newvalue): + self._set_file_handler(self.error_log, newvalue) + error_file = property(_get_error_file, _set_error_file, + doc="""The filename for self.error_log. + + If you set this to a string, it'll add the appropriate FileHandler for + you. If you set it to ``None`` or ``''``, it will remove the handler. + """) + + def _get_access_file(self): + h = self._get_builtin_handler(self.access_log, "file") + if h: + return h.baseFilename + return '' + def _set_access_file(self, newvalue): + self._set_file_handler(self.access_log, newvalue) + access_file = property(_get_access_file, _set_access_file, + doc="""The filename for self.access_log. + + If you set this to a string, it'll add the appropriate FileHandler for + you. If you set it to ``None`` or ``''``, it will remove the handler. + """) + + # ------------------------- WSGI handlers ------------------------- # + + def _set_wsgi_handler(self, log, enable): + h = self._get_builtin_handler(log, "wsgi") + if enable: + if not h: + h = WSGIErrorHandler() + h.setFormatter(logfmt) + h._cpbuiltin = "wsgi" + log.addHandler(h) + elif h: + log.handlers.remove(h) + + def _get_wsgi(self): + return bool(self._get_builtin_handler(self.error_log, "wsgi")) + + def _set_wsgi(self, newvalue): + self._set_wsgi_handler(self.error_log, newvalue) + wsgi = property(_get_wsgi, _set_wsgi, + doc="""Write errors to wsgi.errors. + + If you set this to True, it'll add the appropriate + :class:`WSGIErrorHandler<cherrypy._cplogging.WSGIErrorHandler>` for you + (which writes errors to ``wsgi.errors``). + If you set it to False, it will remove the handler. + """) + + +class WSGIErrorHandler(logging.Handler): + "A handler class which writes logging records to environ['wsgi.errors']." + + def flush(self): + """Flushes the stream.""" + try: + stream = cherrypy.serving.request.wsgi_environ.get('wsgi.errors') + except (AttributeError, KeyError): + pass + else: + stream.flush() + + def emit(self, record): + """Emit a record.""" + try: + stream = cherrypy.serving.request.wsgi_environ.get('wsgi.errors') + except (AttributeError, KeyError): + pass + else: + try: + msg = self.format(record) + fs = "%s\n" + import types + if not hasattr(types, "UnicodeType"): #if no unicode support... + stream.write(fs % msg) + else: + try: + stream.write(fs % msg) + except UnicodeError: + stream.write(fs % msg.encode("UTF-8")) + self.flush() + except: + self.handleError(record) diff --git a/cherrypy/_cpmodpy.py b/cherrypy/_cpmodpy.py new file mode 100755 index 0000000..ba2ab22 --- /dev/null +++ b/cherrypy/_cpmodpy.py @@ -0,0 +1,333 @@ +"""Native adapter for serving CherryPy via mod_python + +Basic usage: + +########################################## +# Application in a module called myapp.py +########################################## + +import cherrypy + +class Root: + @cherrypy.expose + def index(self): + return 'Hi there, Ho there, Hey there' + + +# We will use this method from the mod_python configuration +# as the entry point to our application +def setup_server(): + cherrypy.tree.mount(Root()) + cherrypy.config.update({'environment': 'production', + 'log.screen': False, + 'show_tracebacks': False}) + +########################################## +# mod_python settings for apache2 +# This should reside in your httpd.conf +# or a file that will be loaded at +# apache startup +########################################## + +# Start +DocumentRoot "/" +Listen 8080 +LoadModule python_module /usr/lib/apache2/modules/mod_python.so + +<Location "/"> + PythonPath "sys.path+['/path/to/my/application']" + SetHandler python-program + PythonHandler cherrypy._cpmodpy::handler + PythonOption cherrypy.setup myapp::setup_server + PythonDebug On +</Location> +# End + +The actual path to your mod_python.so is dependent on your +environment. In this case we suppose a global mod_python +installation on a Linux distribution such as Ubuntu. + +We do set the PythonPath configuration setting so that +your application can be found by from the user running +the apache2 instance. Of course if your application +resides in the global site-package this won't be needed. + +Then restart apache2 and access http://127.0.0.1:8080 +""" + +import logging +import sys + +import cherrypy +from cherrypy._cpcompat import BytesIO, copyitems, ntob +from cherrypy._cperror import format_exc, bare_error +from cherrypy.lib import httputil + + +# ------------------------------ Request-handling + + + +def setup(req): + from mod_python import apache + + # Run any setup functions defined by a "PythonOption cherrypy.setup" directive. + options = req.get_options() + if 'cherrypy.setup' in options: + for function in options['cherrypy.setup'].split(): + atoms = function.split('::', 1) + if len(atoms) == 1: + mod = __import__(atoms[0], globals(), locals()) + else: + modname, fname = atoms + mod = __import__(modname, globals(), locals(), [fname]) + func = getattr(mod, fname) + func() + + cherrypy.config.update({'log.screen': False, + "tools.ignore_headers.on": True, + "tools.ignore_headers.headers": ['Range'], + }) + + engine = cherrypy.engine + if hasattr(engine, "signal_handler"): + engine.signal_handler.unsubscribe() + if hasattr(engine, "console_control_handler"): + engine.console_control_handler.unsubscribe() + engine.autoreload.unsubscribe() + cherrypy.server.unsubscribe() + + def _log(msg, level): + newlevel = apache.APLOG_ERR + if logging.DEBUG >= level: + newlevel = apache.APLOG_DEBUG + elif logging.INFO >= level: + newlevel = apache.APLOG_INFO + elif logging.WARNING >= level: + newlevel = apache.APLOG_WARNING + # On Windows, req.server is required or the msg will vanish. See + # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html. + # Also, "When server is not specified...LogLevel does not apply..." + apache.log_error(msg, newlevel, req.server) + engine.subscribe('log', _log) + + engine.start() + + def cherrypy_cleanup(data): + engine.exit() + try: + # apache.register_cleanup wasn't available until 3.1.4. + apache.register_cleanup(cherrypy_cleanup) + except AttributeError: + req.server.register_cleanup(req, cherrypy_cleanup) + + +class _ReadOnlyRequest: + expose = ('read', 'readline', 'readlines') + def __init__(self, req): + for method in self.expose: + self.__dict__[method] = getattr(req, method) + + +recursive = False + +_isSetUp = False +def handler(req): + from mod_python import apache + try: + global _isSetUp + if not _isSetUp: + setup(req) + _isSetUp = True + + # Obtain a Request object from CherryPy + local = req.connection.local_addr + local = httputil.Host(local[0], local[1], req.connection.local_host or "") + remote = req.connection.remote_addr + remote = httputil.Host(remote[0], remote[1], req.connection.remote_host or "") + + scheme = req.parsed_uri[0] or 'http' + req.get_basic_auth_pw() + + try: + # apache.mpm_query only became available in mod_python 3.1 + q = apache.mpm_query + threaded = q(apache.AP_MPMQ_IS_THREADED) + forked = q(apache.AP_MPMQ_IS_FORKED) + except AttributeError: + bad_value = ("You must provide a PythonOption '%s', " + "either 'on' or 'off', when running a version " + "of mod_python < 3.1") + + threaded = options.get('multithread', '').lower() + if threaded == 'on': + threaded = True + elif threaded == 'off': + threaded = False + else: + raise ValueError(bad_value % "multithread") + + forked = options.get('multiprocess', '').lower() + if forked == 'on': + forked = True + elif forked == 'off': + forked = False + else: + raise ValueError(bad_value % "multiprocess") + + sn = cherrypy.tree.script_name(req.uri or "/") + if sn is None: + send_response(req, '404 Not Found', [], '') + else: + app = cherrypy.tree.apps[sn] + method = req.method + path = req.uri + qs = req.args or "" + reqproto = req.protocol + headers = copyitems(req.headers_in) + rfile = _ReadOnlyRequest(req) + prev = None + + try: + redirections = [] + while True: + request, response = app.get_serving(local, remote, scheme, + "HTTP/1.1") + request.login = req.user + request.multithread = bool(threaded) + request.multiprocess = bool(forked) + request.app = app + request.prev = prev + + # Run the CherryPy Request object and obtain the response + try: + request.run(method, path, qs, reqproto, headers, rfile) + break + except cherrypy.InternalRedirect: + ir = sys.exc_info()[1] + app.release_serving() + prev = request + + if not recursive: + if ir.path in redirections: + raise RuntimeError("InternalRedirector visited the " + "same URL twice: %r" % ir.path) + else: + # Add the *previous* path_info + qs to redirections. + if qs: + qs = "?" + qs + redirections.append(sn + path + qs) + + # Munge environment and try again. + method = "GET" + path = ir.path + qs = ir.query_string + rfile = BytesIO() + + send_response(req, response.status, response.header_list, + response.body, response.stream) + finally: + app.release_serving() + except: + tb = format_exc() + cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) + s, h, b = bare_error() + send_response(req, s, h, b) + return apache.OK + + +def send_response(req, status, headers, body, stream=False): + # Set response status + req.status = int(status[:3]) + + # Set response headers + req.content_type = "text/plain" + for header, value in headers: + if header.lower() == 'content-type': + req.content_type = value + continue + req.headers_out.add(header, value) + + if stream: + # Flush now so the status and headers are sent immediately. + req.flush() + + # Set response body + if isinstance(body, basestring): + req.write(body) + else: + for seg in body: + req.write(seg) + + + +# --------------- Startup tools for CherryPy + mod_python --------------- # + + +import os +import re + + +def read_process(cmd, args=""): + fullcmd = "%s %s" % (cmd, args) + pipein, pipeout = os.popen4(fullcmd) + try: + firstline = pipeout.readline() + if (re.search(ntob("(not recognized|No such file|not found)"), firstline, + re.IGNORECASE)): + raise IOError('%s must be on your system path.' % cmd) + output = firstline + pipeout.read() + finally: + pipeout.close() + return output + + +class ModPythonServer(object): + + template = """ +# Apache2 server configuration file for running CherryPy with mod_python. + +DocumentRoot "/" +Listen %(port)s +LoadModule python_module modules/mod_python.so + +<Location %(loc)s> + SetHandler python-program + PythonHandler %(handler)s + PythonDebug On +%(opts)s +</Location> +""" + + def __init__(self, loc="/", port=80, opts=None, apache_path="apache", + handler="cherrypy._cpmodpy::handler"): + self.loc = loc + self.port = port + self.opts = opts + self.apache_path = apache_path + self.handler = handler + + def start(self): + opts = "".join([" PythonOption %s %s\n" % (k, v) + for k, v in self.opts]) + conf_data = self.template % {"port": self.port, + "loc": self.loc, + "opts": opts, + "handler": self.handler, + } + + mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") + f = open(mpconf, 'wb') + try: + f.write(conf_data) + finally: + f.close() + + response = read_process(self.apache_path, "-k start -f %s" % mpconf) + self.ready = True + return response + + def stop(self): + os.popen("apache -k stop") + self.ready = False + diff --git a/cherrypy/_cpnative_server.py b/cherrypy/_cpnative_server.py new file mode 100755 index 0000000..57f715a --- /dev/null +++ b/cherrypy/_cpnative_server.py @@ -0,0 +1,149 @@ +"""Native adapter for serving CherryPy via its builtin server.""" + +import logging +import sys + +import cherrypy +from cherrypy._cpcompat import BytesIO +from cherrypy._cperror import format_exc, bare_error +from cherrypy.lib import httputil +from cherrypy import wsgiserver + + +class NativeGateway(wsgiserver.Gateway): + + recursive = False + + def respond(self): + req = self.req + try: + # Obtain a Request object from CherryPy + local = req.server.bind_addr + local = httputil.Host(local[0], local[1], "") + remote = req.conn.remote_addr, req.conn.remote_port + remote = httputil.Host(remote[0], remote[1], "") + + scheme = req.scheme + sn = cherrypy.tree.script_name(req.uri or "/") + if sn is None: + self.send_response('404 Not Found', [], ['']) + else: + app = cherrypy.tree.apps[sn] + method = req.method + path = req.path + qs = req.qs or "" + headers = req.inheaders.items() + rfile = req.rfile + prev = None + + try: + redirections = [] + while True: + request, response = app.get_serving( + local, remote, scheme, "HTTP/1.1") + request.multithread = True + request.multiprocess = False + request.app = app + request.prev = prev + + # Run the CherryPy Request object and obtain the response + try: + request.run(method, path, qs, req.request_protocol, headers, rfile) + break + except cherrypy.InternalRedirect: + ir = sys.exc_info()[1] + app.release_serving() + prev = request + + if not self.recursive: + if ir.path in redirections: + raise RuntimeError("InternalRedirector visited the " + "same URL twice: %r" % ir.path) + else: + # Add the *previous* path_info + qs to redirections. + if qs: + qs = "?" + qs + redirections.append(sn + path + qs) + + # Munge environment and try again. + method = "GET" + path = ir.path + qs = ir.query_string + rfile = BytesIO() + + self.send_response( + response.output_status, response.header_list, + response.body) + finally: + app.release_serving() + except: + tb = format_exc() + #print tb + cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) + s, h, b = bare_error() + self.send_response(s, h, b) + + def send_response(self, status, headers, body): + req = self.req + + # Set response status + req.status = str(status or "500 Server Error") + + # Set response headers + for header, value in headers: + req.outheaders.append((header, value)) + if (req.ready and not req.sent_headers): + req.sent_headers = True + req.send_headers() + + # Set response body + for seg in body: + req.write(seg) + + +class CPHTTPServer(wsgiserver.HTTPServer): + """Wrapper for wsgiserver.HTTPServer. + + wsgiserver has been designed to not reference CherryPy in any way, + so that it can be used in other frameworks and applications. + Therefore, we wrap it here, so we can apply some attributes + from config -> cherrypy.server -> HTTPServer. + """ + + def __init__(self, server_adapter=cherrypy.server): + self.server_adapter = server_adapter + + server_name = (self.server_adapter.socket_host or + self.server_adapter.socket_file or + None) + + wsgiserver.HTTPServer.__init__( + self, server_adapter.bind_addr, NativeGateway, + minthreads=server_adapter.thread_pool, + maxthreads=server_adapter.thread_pool_max, + server_name=server_name) + + self.max_request_header_size = self.server_adapter.max_request_header_size or 0 + self.max_request_body_size = self.server_adapter.max_request_body_size or 0 + self.request_queue_size = self.server_adapter.socket_queue_size + self.timeout = self.server_adapter.socket_timeout + self.shutdown_timeout = self.server_adapter.shutdown_timeout + self.protocol = self.server_adapter.protocol_version + self.nodelay = self.server_adapter.nodelay + + ssl_module = self.server_adapter.ssl_module or 'pyopenssl' + if self.server_adapter.ssl_context: + adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + self.ssl_adapter = adapter_class( + self.server_adapter.ssl_certificate, + self.server_adapter.ssl_private_key, + self.server_adapter.ssl_certificate_chain) + self.ssl_adapter.context = self.server_adapter.ssl_context + elif self.server_adapter.ssl_certificate: + adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + self.ssl_adapter = adapter_class( + self.server_adapter.ssl_certificate, + self.server_adapter.ssl_private_key, + self.server_adapter.ssl_certificate_chain) + + diff --git a/cherrypy/_cpreqbody.py b/cherrypy/_cpreqbody.py new file mode 100755 index 0000000..1b0496e --- /dev/null +++ b/cherrypy/_cpreqbody.py @@ -0,0 +1,941 @@ +"""Request body processing for CherryPy. + +.. versionadded:: 3.2 + +Application authors have complete control over the parsing of HTTP request +entities. In short, :attr:`cherrypy.request.body<cherrypy._cprequest.Request.body>` +is now always set to an instance of :class:`RequestBody<cherrypy._cpreqbody.RequestBody>`, +and *that* class is a subclass of :class:`Entity<cherrypy._cpreqbody.Entity>`. + +When an HTTP request includes an entity body, it is often desirable to +provide that information to applications in a form other than the raw bytes. +Different content types demand different approaches. Examples: + + * For a GIF file, we want the raw bytes in a stream. + * An HTML form is better parsed into its component fields, and each text field + decoded from bytes to unicode. + * A JSON body should be deserialized into a Python dict or list. + +When the request contains a Content-Type header, the media type is used as a +key to look up a value in the +:attr:`request.body.processors<cherrypy._cpreqbody.Entity.processors>` dict. +If the full media +type is not found, then the major type is tried; for example, if no processor +is found for the 'image/jpeg' type, then we look for a processor for the 'image' +types altogether. If neither the full type nor the major type has a matching +processor, then a default processor is used +(:func:`default_proc<cherrypy._cpreqbody.Entity.default_proc>`). For most +types, this means no processing is done, and the body is left unread as a +raw byte stream. Processors are configurable in an 'on_start_resource' hook. + +Some processors, especially those for the 'text' types, attempt to decode bytes +to unicode. If the Content-Type request header includes a 'charset' parameter, +this is used to decode the entity. Otherwise, one or more default charsets may +be attempted, although this decision is up to each processor. If a processor +successfully decodes an Entity or Part, it should set the +:attr:`charset<cherrypy._cpreqbody.Entity.charset>` attribute +on the Entity or Part to the name of the successful charset, so that +applications can easily re-encode or transcode the value if they wish. + +If the Content-Type of the request entity is of major type 'multipart', then +the above parsing process, and possibly a decoding process, is performed for +each part. + +For both the full entity and multipart parts, a Content-Disposition header may +be used to fill :attr:`name<cherrypy._cpreqbody.Entity.name>` and +:attr:`filename<cherrypy._cpreqbody.Entity.filename>` attributes on the +request.body or the Part. + +.. _custombodyprocessors: + +Custom Processors +================= + +You can add your own processors for any specific or major MIME type. Simply add +it to the :attr:`processors<cherrypy._cprequest.Entity.processors>` dict in a +hook/tool that runs at ``on_start_resource`` or ``before_request_body``. +Here's the built-in JSON tool for an example:: + + def json_in(force=True, debug=False): + request = cherrypy.serving.request + def json_processor(entity): + \"""Read application/json data into request.json.\""" + if not entity.headers.get("Content-Length", ""): + raise cherrypy.HTTPError(411) + + body = entity.fp.read() + try: + request.json = json_decode(body) + except ValueError: + raise cherrypy.HTTPError(400, 'Invalid JSON document') + if force: + request.body.processors.clear() + request.body.default_proc = cherrypy.HTTPError( + 415, 'Expected an application/json content type') + request.body.processors['application/json'] = json_processor + +We begin by defining a new ``json_processor`` function to stick in the ``processors`` +dictionary. All processor functions take a single argument, the ``Entity`` instance +they are to process. It will be called whenever a request is received (for those +URI's where the tool is turned on) which has a ``Content-Type`` of +"application/json". + +First, it checks for a valid ``Content-Length`` (raising 411 if not valid), then +reads the remaining bytes on the socket. The ``fp`` object knows its own length, so +it won't hang waiting for data that never arrives. It will return when all data +has been read. Then, we decode those bytes using Python's built-in ``json`` module, +and stick the decoded result onto ``request.json`` . If it cannot be decoded, we +raise 400. + +If the "force" argument is True (the default), the ``Tool`` clears the ``processors`` +dict so that request entities of other ``Content-Types`` aren't parsed at all. Since +there's no entry for those invalid MIME types, the ``default_proc`` method of ``cherrypy.request.body`` +is called. But this does nothing by default (usually to provide the page handler an opportunity to handle it.) +But in our case, we want to raise 415, so we replace ``request.body.default_proc`` +with the error (``HTTPError`` instances, when called, raise themselves). + +If we were defining a custom processor, we can do so without making a ``Tool``. Just add the config entry:: + + request.body.processors = {'application/json': json_processor} + +Note that you can only replace the ``processors`` dict wholesale this way, not update the existing one. +""" + +import re +import sys +import tempfile +from urllib import unquote_plus + +import cherrypy +from cherrypy._cpcompat import basestring, ntob, ntou +from cherrypy.lib import httputil + + +# -------------------------------- Processors -------------------------------- # + +def process_urlencoded(entity): + """Read application/x-www-form-urlencoded data into entity.params.""" + qs = entity.fp.read() + for charset in entity.attempt_charsets: + try: + params = {} + for aparam in qs.split(ntob('&')): + for pair in aparam.split(ntob(';')): + if not pair: + continue + + atoms = pair.split(ntob('='), 1) + if len(atoms) == 1: + atoms.append(ntob('')) + + key = unquote_plus(atoms[0]).decode(charset) + value = unquote_plus(atoms[1]).decode(charset) + + if key in params: + if not isinstance(params[key], list): + params[key] = [params[key]] + params[key].append(value) + else: + params[key] = value + except UnicodeDecodeError: + pass + else: + entity.charset = charset + break + else: + raise cherrypy.HTTPError( + 400, "The request entity could not be decoded. The following " + "charsets were attempted: %s" % repr(entity.attempt_charsets)) + + # Now that all values have been successfully parsed and decoded, + # apply them to the entity.params dict. + for key, value in params.items(): + if key in entity.params: + if not isinstance(entity.params[key], list): + entity.params[key] = [entity.params[key]] + entity.params[key].append(value) + else: + entity.params[key] = value + + +def process_multipart(entity): + """Read all multipart parts into entity.parts.""" + ib = "" + if 'boundary' in entity.content_type.params: + # http://tools.ietf.org/html/rfc2046#section-5.1.1 + # "The grammar for parameters on the Content-type field is such that it + # is often necessary to enclose the boundary parameter values in quotes + # on the Content-type line" + ib = entity.content_type.params['boundary'].strip('"') + + if not re.match("^[ -~]{0,200}[!-~]$", ib): + raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) + + ib = ('--' + ib).encode('ascii') + + # Find the first marker + while True: + b = entity.readline() + if not b: + return + + b = b.strip() + if b == ib: + break + + # Read all parts + while True: + part = entity.part_class.from_fp(entity.fp, ib) + entity.parts.append(part) + part.process() + if part.fp.done: + break + +def process_multipart_form_data(entity): + """Read all multipart/form-data parts into entity.parts or entity.params.""" + process_multipart(entity) + + kept_parts = [] + for part in entity.parts: + if part.name is None: + kept_parts.append(part) + else: + if part.filename is None: + # It's a regular field + value = part.fullvalue() + else: + # It's a file upload. Retain the whole part so consumer code + # has access to its .file and .filename attributes. + value = part + + if part.name in entity.params: + if not isinstance(entity.params[part.name], list): + entity.params[part.name] = [entity.params[part.name]] + entity.params[part.name].append(value) + else: + entity.params[part.name] = value + + entity.parts = kept_parts + +def _old_process_multipart(entity): + """The behavior of 3.2 and lower. Deprecated and will be changed in 3.3.""" + process_multipart(entity) + + params = entity.params + + for part in entity.parts: + if part.name is None: + key = ntou('parts') + else: + key = part.name + + if part.filename is None: + # It's a regular field + value = part.fullvalue() + else: + # It's a file upload. Retain the whole part so consumer code + # has access to its .file and .filename attributes. + value = part + + if key in params: + if not isinstance(params[key], list): + params[key] = [params[key]] + params[key].append(value) + else: + params[key] = value + + + +# --------------------------------- Entities --------------------------------- # + + +class Entity(object): + """An HTTP request body, or MIME multipart body. + + This class collects information about the HTTP request entity. When a + given entity is of MIME type "multipart", each part is parsed into its own + Entity instance, and the set of parts stored in + :attr:`entity.parts<cherrypy._cpreqbody.Entity.parts>`. + + Between the ``before_request_body`` and ``before_handler`` tools, CherryPy + tries to process the request body (if any) by calling + :func:`request.body.process<cherrypy._cpreqbody.RequestBody.process`. + This uses the ``content_type`` of the Entity to look up a suitable processor + in :attr:`Entity.processors<cherrypy._cpreqbody.Entity.processors>`, a dict. + If a matching processor cannot be found for the complete Content-Type, + it tries again using the major type. For example, if a request with an + entity of type "image/jpeg" arrives, but no processor can be found for + that complete type, then one is sought for the major type "image". If a + processor is still not found, then the + :func:`default_proc<cherrypy._cpreqbody.Entity.default_proc>` method of the + Entity is called (which does nothing by default; you can override this too). + + CherryPy includes processors for the "application/x-www-form-urlencoded" + type, the "multipart/form-data" type, and the "multipart" major type. + CherryPy 3.2 processes these types almost exactly as older versions. + Parts are passed as arguments to the page handler using their + ``Content-Disposition.name`` if given, otherwise in a generic "parts" + argument. Each such part is either a string, or the + :class:`Part<cherrypy._cpreqbody.Part>` itself if it's a file. (In this + case it will have ``file`` and ``filename`` attributes, or possibly a + ``value`` attribute). Each Part is itself a subclass of + Entity, and has its own ``process`` method and ``processors`` dict. + + There is a separate processor for the "multipart" major type which is more + flexible, and simply stores all multipart parts in + :attr:`request.body.parts<cherrypy._cpreqbody.Entity.parts>`. You can + enable it with:: + + cherrypy.request.body.processors['multipart'] = _cpreqbody.process_multipart + + in an ``on_start_resource`` tool. + """ + + # http://tools.ietf.org/html/rfc2046#section-4.1.2: + # "The default character set, which must be assumed in the + # absence of a charset parameter, is US-ASCII." + # However, many browsers send data in utf-8 with no charset. + attempt_charsets = ['utf-8'] + """A list of strings, each of which should be a known encoding. + + When the Content-Type of the request body warrants it, each of the given + encodings will be tried in order. The first one to successfully decode the + entity without raising an error is stored as + :attr:`entity.charset<cherrypy._cpreqbody.Entity.charset>`. This defaults + to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by + `HTTP/1.1 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1>`_), + but ``['us-ascii', 'utf-8']`` for multipart parts. + """ + + charset = None + """The successful decoding; see "attempt_charsets" above.""" + + content_type = None + """The value of the Content-Type request header. + + If the Entity is part of a multipart payload, this will be the Content-Type + given in the MIME headers for this part. + """ + + default_content_type = 'application/x-www-form-urlencoded' + """This defines a default ``Content-Type`` to use if no Content-Type header + is given. The empty string is used for RequestBody, which results in the + request body not being read or parsed at all. This is by design; a missing + ``Content-Type`` header in the HTTP request entity is an error at best, + and a security hole at worst. For multipart parts, however, the MIME spec + declares that a part with no Content-Type defaults to "text/plain" + (see :class:`Part<cherrypy._cpreqbody.Part>`). + """ + + filename = None + """The ``Content-Disposition.filename`` header, if available.""" + + fp = None + """The readable socket file object.""" + + headers = None + """A dict of request/multipart header names and values. + + This is a copy of the ``request.headers`` for the ``request.body``; + for multipart parts, it is the set of headers for that part. + """ + + length = None + """The value of the ``Content-Length`` header, if provided.""" + + name = None + """The "name" parameter of the ``Content-Disposition`` header, if any.""" + + params = None + """ + If the request Content-Type is 'application/x-www-form-urlencoded' or + multipart, this will be a dict of the params pulled from the entity + body; that is, it will be the portion of request.params that come + from the message body (sometimes called "POST params", although they + can be sent with various HTTP method verbs). This value is set between + the 'before_request_body' and 'before_handler' hooks (assuming that + process_request_body is True).""" + + processors = {'application/x-www-form-urlencoded': process_urlencoded, + 'multipart/form-data': process_multipart_form_data, + 'multipart': process_multipart, + } + """A dict of Content-Type names to processor methods.""" + + parts = None + """A list of Part instances if ``Content-Type`` is of major type "multipart".""" + + part_class = None + """The class used for multipart parts. + + You can replace this with custom subclasses to alter the processing of + multipart parts. + """ + + def __init__(self, fp, headers, params=None, parts=None): + # Make an instance-specific copy of the class processors + # so Tools, etc. can replace them per-request. + self.processors = self.processors.copy() + + self.fp = fp + self.headers = headers + + if params is None: + params = {} + self.params = params + + if parts is None: + parts = [] + self.parts = parts + + # Content-Type + self.content_type = headers.elements('Content-Type') + if self.content_type: + self.content_type = self.content_type[0] + else: + self.content_type = httputil.HeaderElement.from_str( + self.default_content_type) + + # Copy the class 'attempt_charsets', prepending any Content-Type charset + dec = self.content_type.params.get("charset", None) + if dec: + #dec = dec.decode('ISO-8859-1') + self.attempt_charsets = [dec] + [c for c in self.attempt_charsets + if c != dec] + else: + self.attempt_charsets = self.attempt_charsets[:] + + # Length + self.length = None + clen = headers.get('Content-Length', None) + # If Transfer-Encoding is 'chunked', ignore any Content-Length. + if clen is not None and 'chunked' not in headers.get('Transfer-Encoding', ''): + try: + self.length = int(clen) + except ValueError: + pass + + # Content-Disposition + self.name = None + self.filename = None + disp = headers.elements('Content-Disposition') + if disp: + disp = disp[0] + if 'name' in disp.params: + self.name = disp.params['name'] + if self.name.startswith('"') and self.name.endswith('"'): + self.name = self.name[1:-1] + if 'filename' in disp.params: + self.filename = disp.params['filename'] + if self.filename.startswith('"') and self.filename.endswith('"'): + self.filename = self.filename[1:-1] + + # The 'type' attribute is deprecated in 3.2; remove it in 3.3. + type = property(lambda self: self.content_type, + doc="""A deprecated alias for :attr:`content_type<cherrypy._cpreqbody.Entity.content_type>`.""") + + def read(self, size=None, fp_out=None): + return self.fp.read(size, fp_out) + + def readline(self, size=None): + return self.fp.readline(size) + + def readlines(self, sizehint=None): + return self.fp.readlines(sizehint) + + def __iter__(self): + return self + + def next(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def read_into_file(self, fp_out=None): + """Read the request body into fp_out (or make_file() if None). Return fp_out.""" + if fp_out is None: + fp_out = self.make_file() + self.read(fp_out=fp_out) + return fp_out + + def make_file(self): + """Return a file-like object into which the request body will be read. + + By default, this will return a TemporaryFile. Override as needed. + See also :attr:`cherrypy._cpreqbody.Part.maxrambytes`.""" + return tempfile.TemporaryFile() + + def fullvalue(self): + """Return this entity as a string, whether stored in a file or not.""" + if self.file: + # It was stored in a tempfile. Read it. + self.file.seek(0) + value = self.file.read() + self.file.seek(0) + else: + value = self.value + return value + + def process(self): + """Execute the best-match processor for the given media type.""" + proc = None + ct = self.content_type.value + try: + proc = self.processors[ct] + except KeyError: + toptype = ct.split('/', 1)[0] + try: + proc = self.processors[toptype] + except KeyError: + pass + if proc is None: + self.default_proc() + else: + proc(self) + + def default_proc(self): + """Called if a more-specific processor is not found for the ``Content-Type``.""" + # Leave the fp alone for someone else to read. This works fine + # for request.body, but the Part subclasses need to override this + # so they can move on to the next part. + pass + + +class Part(Entity): + """A MIME part entity, part of a multipart entity.""" + + # "The default character set, which must be assumed in the absence of a + # charset parameter, is US-ASCII." + attempt_charsets = ['us-ascii', 'utf-8'] + """A list of strings, each of which should be a known encoding. + + When the Content-Type of the request body warrants it, each of the given + encodings will be tried in order. The first one to successfully decode the + entity without raising an error is stored as + :attr:`entity.charset<cherrypy._cpreqbody.Entity.charset>`. This defaults + to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by + `HTTP/1.1 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1>`_), + but ``['us-ascii', 'utf-8']`` for multipart parts. + """ + + boundary = None + """The MIME multipart boundary.""" + + default_content_type = 'text/plain' + """This defines a default ``Content-Type`` to use if no Content-Type header + is given. The empty string is used for RequestBody, which results in the + request body not being read or parsed at all. This is by design; a missing + ``Content-Type`` header in the HTTP request entity is an error at best, + and a security hole at worst. For multipart parts, however (this class), + the MIME spec declares that a part with no Content-Type defaults to + "text/plain". + """ + + # This is the default in stdlib cgi. We may want to increase it. + maxrambytes = 1000 + """The threshold of bytes after which point the ``Part`` will store its data + in a file (generated by :func:`make_file<cherrypy._cprequest.Entity.make_file>`) + instead of a string. Defaults to 1000, just like the :mod:`cgi` module in + Python's standard library. + """ + + def __init__(self, fp, headers, boundary): + Entity.__init__(self, fp, headers) + self.boundary = boundary + self.file = None + self.value = None + + def from_fp(cls, fp, boundary): + headers = cls.read_headers(fp) + return cls(fp, headers, boundary) + from_fp = classmethod(from_fp) + + def read_headers(cls, fp): + headers = httputil.HeaderMap() + while True: + line = fp.readline() + if not line: + # No more data--illegal end of headers + raise EOFError("Illegal end of headers.") + + if line == ntob('\r\n'): + # Normal end of headers + break + if not line.endswith(ntob('\r\n')): + raise ValueError("MIME requires CRLF terminators: %r" % line) + + if line[0] in ntob(' \t'): + # It's a continuation line. + v = line.strip().decode('ISO-8859-1') + else: + k, v = line.split(ntob(":"), 1) + k = k.strip().decode('ISO-8859-1') + v = v.strip().decode('ISO-8859-1') + + existing = headers.get(k) + if existing: + v = ", ".join((existing, v)) + headers[k] = v + + return headers + read_headers = classmethod(read_headers) + + def read_lines_to_boundary(self, fp_out=None): + """Read bytes from self.fp and return or write them to a file. + + If the 'fp_out' argument is None (the default), all bytes read are + returned in a single byte string. + + If the 'fp_out' argument is not None, it must be a file-like object that + supports the 'write' method; all bytes read will be written to the fp, + and that fp is returned. + """ + endmarker = self.boundary + ntob("--") + delim = ntob("") + prev_lf = True + lines = [] + seen = 0 + while True: + line = self.fp.readline(1<<16) + if not line: + raise EOFError("Illegal end of multipart body.") + if line.startswith(ntob("--")) and prev_lf: + strippedline = line.strip() + if strippedline == self.boundary: + break + if strippedline == endmarker: + self.fp.finish() + break + + line = delim + line + + if line.endswith(ntob("\r\n")): + delim = ntob("\r\n") + line = line[:-2] + prev_lf = True + elif line.endswith(ntob("\n")): + delim = ntob("\n") + line = line[:-1] + prev_lf = True + else: + delim = ntob("") + prev_lf = False + + if fp_out is None: + lines.append(line) + seen += len(line) + if seen > self.maxrambytes: + fp_out = self.make_file() + for line in lines: + fp_out.write(line) + else: + fp_out.write(line) + + if fp_out is None: + result = ntob('').join(lines) + for charset in self.attempt_charsets: + try: + result = result.decode(charset) + except UnicodeDecodeError: + pass + else: + self.charset = charset + return result + else: + raise cherrypy.HTTPError( + 400, "The request entity could not be decoded. The following " + "charsets were attempted: %s" % repr(self.attempt_charsets)) + else: + fp_out.seek(0) + return fp_out + + def default_proc(self): + """Called if a more-specific processor is not found for the ``Content-Type``.""" + if self.filename: + # Always read into a file if a .filename was given. + self.file = self.read_into_file() + else: + result = self.read_lines_to_boundary() + if isinstance(result, basestring): + self.value = result + else: + self.file = result + + def read_into_file(self, fp_out=None): + """Read the request body into fp_out (or make_file() if None). Return fp_out.""" + if fp_out is None: + fp_out = self.make_file() + self.read_lines_to_boundary(fp_out=fp_out) + return fp_out + +Entity.part_class = Part + + +class Infinity(object): + def __cmp__(self, other): + return 1 + def __sub__(self, other): + return self +inf = Infinity() + + +comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', 'Connection', + 'Content-Encoding', 'Content-Language', 'Expect', 'If-Match', + 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'Te', 'Trailer', + 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', 'Www-Authenticate'] + + +class SizedReader: + + def __init__(self, fp, length, maxbytes, bufsize=8192, has_trailers=False): + # Wrap our fp in a buffer so peek() works + self.fp = fp + self.length = length + self.maxbytes = maxbytes + self.buffer = ntob('') + self.bufsize = bufsize + self.bytes_read = 0 + self.done = False + self.has_trailers = has_trailers + + def read(self, size=None, fp_out=None): + """Read bytes from the request body and return or write them to a file. + + A number of bytes less than or equal to the 'size' argument are read + off the socket. The actual number of bytes read are tracked in + self.bytes_read. The number may be smaller than 'size' when 1) the + client sends fewer bytes, 2) the 'Content-Length' request header + specifies fewer bytes than requested, or 3) the number of bytes read + exceeds self.maxbytes (in which case, 413 is raised). + + If the 'fp_out' argument is None (the default), all bytes read are + returned in a single byte string. + + If the 'fp_out' argument is not None, it must be a file-like object that + supports the 'write' method; all bytes read will be written to the fp, + and None is returned. + """ + + if self.length is None: + if size is None: + remaining = inf + else: + remaining = size + else: + remaining = self.length - self.bytes_read + if size and size < remaining: + remaining = size + if remaining == 0: + self.finish() + if fp_out is None: + return ntob('') + else: + return None + + chunks = [] + + # Read bytes from the buffer. + if self.buffer: + if remaining is inf: + data = self.buffer + self.buffer = ntob('') + else: + data = self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + datalen = len(data) + remaining -= datalen + + # Check lengths. + self.bytes_read += datalen + if self.maxbytes and self.bytes_read > self.maxbytes: + raise cherrypy.HTTPError(413) + + # Store the data. + if fp_out is None: + chunks.append(data) + else: + fp_out.write(data) + + # Read bytes from the socket. + while remaining > 0: + chunksize = min(remaining, self.bufsize) + try: + data = self.fp.read(chunksize) + except Exception: + e = sys.exc_info()[1] + if e.__class__.__name__ == 'MaxSizeExceeded': + # Post data is too big + raise cherrypy.HTTPError( + 413, "Maximum request length: %r" % e.args[1]) + else: + raise + if not data: + self.finish() + break + datalen = len(data) + remaining -= datalen + + # Check lengths. + self.bytes_read += datalen + if self.maxbytes and self.bytes_read > self.maxbytes: + raise cherrypy.HTTPError(413) + + # Store the data. + if fp_out is None: + chunks.append(data) + else: + fp_out.write(data) + + if fp_out is None: + return ntob('').join(chunks) + + def readline(self, size=None): + """Read a line from the request body and return it.""" + chunks = [] + while size is None or size > 0: + chunksize = self.bufsize + if size is not None and size < self.bufsize: + chunksize = size + data = self.read(chunksize) + if not data: + break + pos = data.find(ntob('\n')) + 1 + if pos: + chunks.append(data[:pos]) + remainder = data[pos:] + self.buffer += remainder + self.bytes_read -= len(remainder) + break + else: + chunks.append(data) + return ntob('').join(chunks) + + def readlines(self, sizehint=None): + """Read lines from the request body and return them.""" + if self.length is not None: + if sizehint is None: + sizehint = self.length - self.bytes_read + else: + sizehint = min(sizehint, self.length - self.bytes_read) + + lines = [] + seen = 0 + while True: + line = self.readline() + if not line: + break + lines.append(line) + seen += len(line) + if seen >= sizehint: + break + return lines + + def finish(self): + self.done = True + if self.has_trailers and hasattr(self.fp, 'read_trailer_lines'): + self.trailers = {} + + try: + for line in self.fp.read_trailer_lines(): + if line[0] in ntob(' \t'): + # It's a continuation line. + v = line.strip() + else: + try: + k, v = line.split(ntob(":"), 1) + except ValueError: + raise ValueError("Illegal header line.") + k = k.strip().title() + v = v.strip() + + if k in comma_separated_headers: + existing = self.trailers.get(envname) + if existing: + v = ntob(", ").join((existing, v)) + self.trailers[k] = v + except Exception: + e = sys.exc_info()[1] + if e.__class__.__name__ == 'MaxSizeExceeded': + # Post data is too big + raise cherrypy.HTTPError( + 413, "Maximum request length: %r" % e.args[1]) + else: + raise + + +class RequestBody(Entity): + """The entity of the HTTP request.""" + + bufsize = 8 * 1024 + """The buffer size used when reading the socket.""" + + # Don't parse the request body at all if the client didn't provide + # a Content-Type header. See http://www.cherrypy.org/ticket/790 + default_content_type = '' + """This defines a default ``Content-Type`` to use if no Content-Type header + is given. The empty string is used for RequestBody, which results in the + request body not being read or parsed at all. This is by design; a missing + ``Content-Type`` header in the HTTP request entity is an error at best, + and a security hole at worst. For multipart parts, however, the MIME spec + declares that a part with no Content-Type defaults to "text/plain" + (see :class:`Part<cherrypy._cpreqbody.Part>`). + """ + + maxbytes = None + """Raise ``MaxSizeExceeded`` if more bytes than this are read from the socket.""" + + def __init__(self, fp, headers, params=None, request_params=None): + Entity.__init__(self, fp, headers, params) + + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 + # When no explicit charset parameter is provided by the + # sender, media subtypes of the "text" type are defined + # to have a default charset value of "ISO-8859-1" when + # received via HTTP. + if self.content_type.value.startswith('text/'): + for c in ('ISO-8859-1', 'iso-8859-1', 'Latin-1', 'latin-1'): + if c in self.attempt_charsets: + break + else: + self.attempt_charsets.append('ISO-8859-1') + + # Temporary fix while deprecating passing .parts as .params. + self.processors['multipart'] = _old_process_multipart + + if request_params is None: + request_params = {} + self.request_params = request_params + + def process(self): + """Process the request entity based on its Content-Type.""" + # "The presence of a message-body in a request is signaled by the + # inclusion of a Content-Length or Transfer-Encoding header field in + # the request's message-headers." + # It is possible to send a POST request with no body, for example; + # however, app developers are responsible in that case to set + # cherrypy.request.process_body to False so this method isn't called. + h = cherrypy.serving.request.headers + if 'Content-Length' not in h and 'Transfer-Encoding' not in h: + raise cherrypy.HTTPError(411) + + self.fp = SizedReader(self.fp, self.length, + self.maxbytes, bufsize=self.bufsize, + has_trailers='Trailer' in h) + super(RequestBody, self).process() + + # Body params should also be a part of the request_params + # add them in here. + request_params = self.request_params + for key, value in self.params.items(): + # Python 2 only: keyword arguments must be byte strings (type 'str'). + if isinstance(key, unicode): + key = key.encode('ISO-8859-1') + + if key in request_params: + if not isinstance(request_params[key], list): + request_params[key] = [request_params[key]] + request_params[key].append(value) + else: + request_params[key] = value diff --git a/cherrypy/_cprequest.py b/cherrypy/_cprequest.py new file mode 100755 index 0000000..ae5e897 --- /dev/null +++ b/cherrypy/_cprequest.py @@ -0,0 +1,926 @@ + +import os +import sys +import time +import warnings + +import cherrypy +from cherrypy._cpcompat import basestring, copykeys, ntob, unicodestr +from cherrypy._cpcompat import SimpleCookie, CookieError +from cherrypy import _cpreqbody, _cpconfig +from cherrypy._cperror import format_exc, bare_error +from cherrypy.lib import httputil, file_generator + + +class Hook(object): + """A callback and its metadata: failsafe, priority, and kwargs.""" + + callback = None + """ + The bare callable that this Hook object is wrapping, which will + be called when the Hook is called.""" + + failsafe = False + """ + If True, the callback is guaranteed to run even if other callbacks + from the same call point raise exceptions.""" + + priority = 50 + """ + Defines the order of execution for a list of Hooks. Priority numbers + should be limited to the closed interval [0, 100], but values outside + this range are acceptable, as are fractional values.""" + + kwargs = {} + """ + A set of keyword arguments that will be passed to the + callable on each call.""" + + def __init__(self, callback, failsafe=None, priority=None, **kwargs): + self.callback = callback + + if failsafe is None: + failsafe = getattr(callback, "failsafe", False) + self.failsafe = failsafe + + if priority is None: + priority = getattr(callback, "priority", 50) + self.priority = priority + + self.kwargs = kwargs + + def __cmp__(self, other): + return cmp(self.priority, other.priority) + + def __call__(self): + """Run self.callback(**self.kwargs).""" + return self.callback(**self.kwargs) + + def __repr__(self): + cls = self.__class__ + return ("%s.%s(callback=%r, failsafe=%r, priority=%r, %s)" + % (cls.__module__, cls.__name__, self.callback, + self.failsafe, self.priority, + ", ".join(['%s=%r' % (k, v) + for k, v in self.kwargs.items()]))) + + +class HookMap(dict): + """A map of call points to lists of callbacks (Hook objects).""" + + def __new__(cls, points=None): + d = dict.__new__(cls) + for p in points or []: + d[p] = [] + return d + + def __init__(self, *a, **kw): + pass + + def attach(self, point, callback, failsafe=None, priority=None, **kwargs): + """Append a new Hook made from the supplied arguments.""" + self[point].append(Hook(callback, failsafe, priority, **kwargs)) + + def run(self, point): + """Execute all registered Hooks (callbacks) for the given point.""" + exc = None + hooks = self[point] + hooks.sort() + for hook in hooks: + # Some hooks are guaranteed to run even if others at + # the same hookpoint fail. We will still log the failure, + # but proceed on to the next hook. The only way + # to stop all processing from one of these hooks is + # to raise SystemExit and stop the whole server. + if exc is None or hook.failsafe: + try: + hook() + except (KeyboardInterrupt, SystemExit): + raise + except (cherrypy.HTTPError, cherrypy.HTTPRedirect, + cherrypy.InternalRedirect): + exc = sys.exc_info()[1] + except: + exc = sys.exc_info()[1] + cherrypy.log(traceback=True, severity=40) + if exc: + raise + + def __copy__(self): + newmap = self.__class__() + # We can't just use 'update' because we want copies of the + # mutable values (each is a list) as well. + for k, v in self.items(): + newmap[k] = v[:] + return newmap + copy = __copy__ + + def __repr__(self): + cls = self.__class__ + return "%s.%s(points=%r)" % (cls.__module__, cls.__name__, copykeys(self)) + + +# Config namespace handlers + +def hooks_namespace(k, v): + """Attach bare hooks declared in config.""" + # Use split again to allow multiple hooks for a single + # hookpoint per path (e.g. "hooks.before_handler.1"). + # Little-known fact you only get from reading source ;) + hookpoint = k.split(".", 1)[0] + if isinstance(v, basestring): + v = cherrypy.lib.attributes(v) + if not isinstance(v, Hook): + v = Hook(v) + cherrypy.serving.request.hooks[hookpoint].append(v) + +def request_namespace(k, v): + """Attach request attributes declared in config.""" + # Provides config entries to set request.body attrs (like attempt_charsets). + if k[:5] == 'body.': + setattr(cherrypy.serving.request.body, k[5:], v) + else: + setattr(cherrypy.serving.request, k, v) + +def response_namespace(k, v): + """Attach response attributes declared in config.""" + # Provides config entries to set default response headers + # http://cherrypy.org/ticket/889 + if k[:8] == 'headers.': + cherrypy.serving.response.headers[k.split('.', 1)[1]] = v + else: + setattr(cherrypy.serving.response, k, v) + +def error_page_namespace(k, v): + """Attach error pages declared in config.""" + if k != 'default': + k = int(k) + cherrypy.serving.request.error_page[k] = v + + +hookpoints = ['on_start_resource', 'before_request_body', + 'before_handler', 'before_finalize', + 'on_end_resource', 'on_end_request', + 'before_error_response', 'after_error_response'] + + +class Request(object): + """An HTTP request. + + This object represents the metadata of an HTTP request message; + that is, it contains attributes which describe the environment + in which the request URL, headers, and body were sent (if you + want tools to interpret the headers and body, those are elsewhere, + mostly in Tools). This 'metadata' consists of socket data, + transport characteristics, and the Request-Line. This object + also contains data regarding the configuration in effect for + the given URL, and the execution plan for generating a response. + """ + + prev = None + """ + The previous Request object (if any). This should be None + unless we are processing an InternalRedirect.""" + + # Conversation/connection attributes + local = httputil.Host("127.0.0.1", 80) + "An httputil.Host(ip, port, hostname) object for the server socket." + + remote = httputil.Host("127.0.0.1", 1111) + "An httputil.Host(ip, port, hostname) object for the client socket." + + scheme = "http" + """ + The protocol used between client and server. In most cases, + this will be either 'http' or 'https'.""" + + server_protocol = "HTTP/1.1" + """ + The HTTP version for which the HTTP server is at least + conditionally compliant.""" + + base = "" + """The (scheme://host) portion of the requested URL. + In some cases (e.g. when proxying via mod_rewrite), this may contain + path segments which cherrypy.url uses when constructing url's, but + which otherwise are ignored by CherryPy. Regardless, this value + MUST NOT end in a slash.""" + + # Request-Line attributes + request_line = "" + """ + The complete Request-Line received from the client. This is a + single string consisting of the request method, URI, and protocol + version (joined by spaces). Any final CRLF is removed.""" + + method = "GET" + """ + Indicates the HTTP method to be performed on the resource identified + by the Request-URI. Common methods include GET, HEAD, POST, PUT, and + DELETE. CherryPy allows any extension method; however, various HTTP + servers and gateways may restrict the set of allowable methods. + CherryPy applications SHOULD restrict the set (on a per-URI basis).""" + + query_string = "" + """ + The query component of the Request-URI, a string of information to be + interpreted by the resource. The query portion of a URI follows the + path component, and is separated by a '?'. For example, the URI + 'http://www.cherrypy.org/wiki?a=3&b=4' has the query component, + 'a=3&b=4'.""" + + query_string_encoding = 'utf8' + """ + The encoding expected for query string arguments after % HEX HEX decoding). + If a query string is provided that cannot be decoded with this encoding, + 404 is raised (since technically it's a different URI). If you want + arbitrary encodings to not error, set this to 'Latin-1'; you can then + encode back to bytes and re-decode to whatever encoding you like later. + """ + + protocol = (1, 1) + """The HTTP protocol version corresponding to the set + of features which should be allowed in the response. If BOTH + the client's request message AND the server's level of HTTP + compliance is HTTP/1.1, this attribute will be the tuple (1, 1). + If either is 1.0, this attribute will be the tuple (1, 0). + Lower HTTP protocol versions are not explicitly supported.""" + + params = {} + """ + A dict which combines query string (GET) and request entity (POST) + variables. This is populated in two stages: GET params are added + before the 'on_start_resource' hook, and POST params are added + between the 'before_request_body' and 'before_handler' hooks.""" + + # Message attributes + header_list = [] + """ + A list of the HTTP request headers as (name, value) tuples. + In general, you should use request.headers (a dict) instead.""" + + headers = httputil.HeaderMap() + """ + A dict-like object containing the request headers. Keys are header + names (in Title-Case format); however, you may get and set them in + a case-insensitive manner. That is, headers['Content-Type'] and + headers['content-type'] refer to the same value. Values are header + values (decoded according to :rfc:`2047` if necessary). See also: + httputil.HeaderMap, httputil.HeaderElement.""" + + cookie = SimpleCookie() + """See help(Cookie).""" + + rfile = None + """ + If the request included an entity (body), it will be available + as a stream in this attribute. However, the rfile will normally + be read for you between the 'before_request_body' hook and the + 'before_handler' hook, and the resulting string is placed into + either request.params or the request.body attribute. + + You may disable the automatic consumption of the rfile by setting + request.process_request_body to False, either in config for the desired + path, or in an 'on_start_resource' or 'before_request_body' hook. + + WARNING: In almost every case, you should not attempt to read from the + rfile stream after CherryPy's automatic mechanism has read it. If you + turn off the automatic parsing of rfile, you should read exactly the + number of bytes specified in request.headers['Content-Length']. + Ignoring either of these warnings may result in a hung request thread + or in corruption of the next (pipelined) request. + """ + + process_request_body = True + """ + If True, the rfile (if any) is automatically read and parsed, + and the result placed into request.params or request.body.""" + + methods_with_bodies = ("POST", "PUT") + """ + A sequence of HTTP methods for which CherryPy will automatically + attempt to read a body from the rfile.""" + + body = None + """ + If the request Content-Type is 'application/x-www-form-urlencoded' + or multipart, this will be None. Otherwise, this will be an instance + of :class:`RequestBody<cherrypy._cpreqbody.RequestBody>` (which you + can .read()); this value is set between the 'before_request_body' and + 'before_handler' hooks (assuming that process_request_body is True).""" + + # Dispatch attributes + dispatch = cherrypy.dispatch.Dispatcher() + """ + The object which looks up the 'page handler' callable and collects + config for the current request based on the path_info, other + request attributes, and the application architecture. The core + calls the dispatcher as early as possible, passing it a 'path_info' + argument. + + The default dispatcher discovers the page handler by matching path_info + to a hierarchical arrangement of objects, starting at request.app.root. + See help(cherrypy.dispatch) for more information.""" + + script_name = "" + """ + The 'mount point' of the application which is handling this request. + + This attribute MUST NOT end in a slash. If the script_name refers to + the root of the URI, it MUST be an empty string (not "/"). + """ + + path_info = "/" + """ + The 'relative path' portion of the Request-URI. This is relative + to the script_name ('mount point') of the application which is + handling this request.""" + + login = None + """ + When authentication is used during the request processing this is + set to 'False' if it failed and to the 'username' value if it succeeded. + The default 'None' implies that no authentication happened.""" + + # Note that cherrypy.url uses "if request.app:" to determine whether + # the call is during a real HTTP request or not. So leave this None. + app = None + """The cherrypy.Application object which is handling this request.""" + + handler = None + """ + The function, method, or other callable which CherryPy will call to + produce the response. The discovery of the handler and the arguments + it will receive are determined by the request.dispatch object. + By default, the handler is discovered by walking a tree of objects + starting at request.app.root, and is then passed all HTTP params + (from the query string and POST body) as keyword arguments.""" + + toolmaps = {} + """ + A nested dict of all Toolboxes and Tools in effect for this request, + of the form: {Toolbox.namespace: {Tool.name: config dict}}.""" + + config = None + """ + A flat dict of all configuration entries which apply to the + current request. These entries are collected from global config, + application config (based on request.path_info), and from handler + config (exactly how is governed by the request.dispatch object in + effect for this request; by default, handler config can be attached + anywhere in the tree between request.app.root and the final handler, + and inherits downward).""" + + is_index = None + """ + This will be True if the current request is mapped to an 'index' + resource handler (also, a 'default' handler if path_info ends with + a slash). The value may be used to automatically redirect the + user-agent to a 'more canonical' URL which either adds or removes + the trailing slash. See cherrypy.tools.trailing_slash.""" + + hooks = HookMap(hookpoints) + """ + A HookMap (dict-like object) of the form: {hookpoint: [hook, ...]}. + Each key is a str naming the hook point, and each value is a list + of hooks which will be called at that hook point during this request. + The list of hooks is generally populated as early as possible (mostly + from Tools specified in config), but may be extended at any time. + See also: _cprequest.Hook, _cprequest.HookMap, and cherrypy.tools.""" + + error_response = cherrypy.HTTPError(500).set_response + """ + The no-arg callable which will handle unexpected, untrapped errors + during request processing. This is not used for expected exceptions + (like NotFound, HTTPError, or HTTPRedirect) which are raised in + response to expected conditions (those should be customized either + via request.error_page or by overriding HTTPError.set_response). + By default, error_response uses HTTPError(500) to return a generic + error response to the user-agent.""" + + error_page = {} + """ + A dict of {error code: response filename or callable} pairs. + + The error code must be an int representing a given HTTP error code, + or the string 'default', which will be used if no matching entry + is found for a given numeric code. + + If a filename is provided, the file should contain a Python string- + formatting template, and can expect by default to receive format + values with the mapping keys %(status)s, %(message)s, %(traceback)s, + and %(version)s. The set of format mappings can be extended by + overriding HTTPError.set_response. + + If a callable is provided, it will be called by default with keyword + arguments 'status', 'message', 'traceback', and 'version', as for a + string-formatting template. The callable must return a string or iterable of + strings which will be set to response.body. It may also override headers or + perform any other processing. + + If no entry is given for an error code, and no 'default' entry exists, + a default template will be used. + """ + + show_tracebacks = True + """ + If True, unexpected errors encountered during request processing will + include a traceback in the response body.""" + + show_mismatched_params = True + """ + If True, mismatched parameters encountered during PageHandler invocation + processing will be included in the response body.""" + + throws = (KeyboardInterrupt, SystemExit, cherrypy.InternalRedirect) + """The sequence of exceptions which Request.run does not trap.""" + + throw_errors = False + """ + If True, Request.run will not trap any errors (except HTTPRedirect and + HTTPError, which are more properly called 'exceptions', not errors).""" + + closed = False + """True once the close method has been called, False otherwise.""" + + stage = None + """ + A string containing the stage reached in the request-handling process. + This is useful when debugging a live server with hung requests.""" + + namespaces = _cpconfig.NamespaceSet( + **{"hooks": hooks_namespace, + "request": request_namespace, + "response": response_namespace, + "error_page": error_page_namespace, + "tools": cherrypy.tools, + }) + + def __init__(self, local_host, remote_host, scheme="http", + server_protocol="HTTP/1.1"): + """Populate a new Request object. + + local_host should be an httputil.Host object with the server info. + remote_host should be an httputil.Host object with the client info. + scheme should be a string, either "http" or "https". + """ + self.local = local_host + self.remote = remote_host + self.scheme = scheme + self.server_protocol = server_protocol + + self.closed = False + + # Put a *copy* of the class error_page into self. + self.error_page = self.error_page.copy() + + # Put a *copy* of the class namespaces into self. + self.namespaces = self.namespaces.copy() + + self.stage = None + + def close(self): + """Run cleanup code. (Core)""" + if not self.closed: + self.closed = True + self.stage = 'on_end_request' + self.hooks.run('on_end_request') + self.stage = 'close' + + def run(self, method, path, query_string, req_protocol, headers, rfile): + """Process the Request. (Core) + + method, path, query_string, and req_protocol should be pulled directly + from the Request-Line (e.g. "GET /path?key=val HTTP/1.0"). + + path + This should be %XX-unquoted, but query_string should not be. + They both MUST be byte strings, not unicode strings. + + headers + A list of (name, value) tuples. + + rfile + A file-like object containing the HTTP request entity. + + When run() is done, the returned object should have 3 attributes: + + * status, e.g. "200 OK" + * header_list, a list of (name, value) tuples + * body, an iterable yielding strings + + Consumer code (HTTP servers) should then access these response + attributes to build the outbound stream. + + """ + response = cherrypy.serving.response + self.stage = 'run' + try: + self.error_response = cherrypy.HTTPError(500).set_response + + self.method = method + path = path or "/" + self.query_string = query_string or '' + self.params = {} + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + rp = int(req_protocol[5]), int(req_protocol[7]) + sp = int(self.server_protocol[5]), int(self.server_protocol[7]) + self.protocol = min(rp, sp) + response.headers.protocol = self.protocol + + # Rebuild first line of the request (e.g. "GET /path HTTP/1.0"). + url = path + if query_string: + url += '?' + query_string + self.request_line = '%s %s %s' % (method, url, req_protocol) + + self.header_list = list(headers) + self.headers = httputil.HeaderMap() + + self.rfile = rfile + self.body = None + + self.cookie = SimpleCookie() + self.handler = None + + # path_info should be the path from the + # app root (script_name) to the handler. + self.script_name = self.app.script_name + self.path_info = pi = path[len(self.script_name):] + + self.stage = 'respond' + self.respond(pi) + + except self.throws: + raise + except: + if self.throw_errors: + raise + else: + # Failure in setup, error handler or finalize. Bypass them. + # Can't use handle_error because we may not have hooks yet. + cherrypy.log(traceback=True, severity=40) + if self.show_tracebacks: + body = format_exc() + else: + body = "" + r = bare_error(body) + response.output_status, response.header_list, response.body = r + + if self.method == "HEAD": + # HEAD requests MUST NOT return a message-body in the response. + response.body = [] + + try: + cherrypy.log.access() + except: + cherrypy.log.error(traceback=True) + + if response.timed_out: + raise cherrypy.TimeoutError() + + return response + + # Uncomment for stage debugging + # stage = property(lambda self: self._stage, lambda self, v: print(v)) + + def respond(self, path_info): + """Generate a response for the resource at self.path_info. (Core)""" + response = cherrypy.serving.response + try: + try: + try: + if self.app is None: + raise cherrypy.NotFound() + + # Get the 'Host' header, so we can HTTPRedirect properly. + self.stage = 'process_headers' + self.process_headers() + + # Make a copy of the class hooks + self.hooks = self.__class__.hooks.copy() + self.toolmaps = {} + + self.stage = 'get_resource' + self.get_resource(path_info) + + self.body = _cpreqbody.RequestBody( + self.rfile, self.headers, request_params=self.params) + + self.namespaces(self.config) + + self.stage = 'on_start_resource' + self.hooks.run('on_start_resource') + + # Parse the querystring + self.stage = 'process_query_string' + self.process_query_string() + + # Process the body + if self.process_request_body: + if self.method not in self.methods_with_bodies: + self.process_request_body = False + self.stage = 'before_request_body' + self.hooks.run('before_request_body') + if self.process_request_body: + self.body.process() + + # Run the handler + self.stage = 'before_handler' + self.hooks.run('before_handler') + if self.handler: + self.stage = 'handler' + response.body = self.handler() + + # Finalize + self.stage = 'before_finalize' + self.hooks.run('before_finalize') + response.finalize() + except (cherrypy.HTTPRedirect, cherrypy.HTTPError): + inst = sys.exc_info()[1] + inst.set_response() + self.stage = 'before_finalize (HTTPError)' + self.hooks.run('before_finalize') + response.finalize() + finally: + self.stage = 'on_end_resource' + self.hooks.run('on_end_resource') + except self.throws: + raise + except: + if self.throw_errors: + raise + self.handle_error() + + def process_query_string(self): + """Parse the query string into Python structures. (Core)""" + try: + p = httputil.parse_query_string( + self.query_string, encoding=self.query_string_encoding) + except UnicodeDecodeError: + raise cherrypy.HTTPError( + 404, "The given query string could not be processed. Query " + "strings for this resource must be encoded with %r." % + self.query_string_encoding) + + # Python 2 only: keyword arguments must be byte strings (type 'str'). + for key, value in p.items(): + if isinstance(key, unicode): + del p[key] + p[key.encode(self.query_string_encoding)] = value + self.params.update(p) + + def process_headers(self): + """Parse HTTP header data into Python structures. (Core)""" + # Process the headers into self.headers + headers = self.headers + for name, value in self.header_list: + # Call title() now (and use dict.__method__(headers)) + # so title doesn't have to be called twice. + name = name.title() + value = value.strip() + + # Warning: if there is more than one header entry for cookies (AFAIK, + # only Konqueror does that), only the last one will remain in headers + # (but they will be correctly stored in request.cookie). + if "=?" in value: + dict.__setitem__(headers, name, httputil.decode_TEXT(value)) + else: + dict.__setitem__(headers, name, value) + + # Handle cookies differently because on Konqueror, multiple + # cookies come on different lines with the same key + if name == 'Cookie': + try: + self.cookie.load(value) + except CookieError: + msg = "Illegal cookie name %s" % value.split('=')[0] + raise cherrypy.HTTPError(400, msg) + + if not dict.__contains__(headers, 'Host'): + # All Internet-based HTTP/1.1 servers MUST respond with a 400 + # (Bad Request) status code to any HTTP/1.1 request message + # which lacks a Host header field. + if self.protocol >= (1, 1): + msg = "HTTP/1.1 requires a 'Host' request header." + raise cherrypy.HTTPError(400, msg) + host = dict.get(headers, 'Host') + if not host: + host = self.local.name or self.local.ip + self.base = "%s://%s" % (self.scheme, host) + + def get_resource(self, path): + """Call a dispatcher (which sets self.handler and .config). (Core)""" + # First, see if there is a custom dispatch at this URI. Custom + # dispatchers can only be specified in app.config, not in _cp_config + # (since custom dispatchers may not even have an app.root). + dispatch = self.app.find_config(path, "request.dispatch", self.dispatch) + + # dispatch() should set self.handler and self.config + dispatch(path) + + def handle_error(self): + """Handle the last unanticipated exception. (Core)""" + try: + self.hooks.run("before_error_response") + if self.error_response: + self.error_response() + self.hooks.run("after_error_response") + cherrypy.serving.response.finalize() + except cherrypy.HTTPRedirect: + inst = sys.exc_info()[1] + inst.set_response() + cherrypy.serving.response.finalize() + + # ------------------------- Properties ------------------------- # + + def _get_body_params(self): + warnings.warn( + "body_params is deprecated in CherryPy 3.2, will be removed in " + "CherryPy 3.3.", + DeprecationWarning + ) + return self.body.params + body_params = property(_get_body_params, + doc= """ + If the request Content-Type is 'application/x-www-form-urlencoded' or + multipart, this will be a dict of the params pulled from the entity + body; that is, it will be the portion of request.params that come + from the message body (sometimes called "POST params", although they + can be sent with various HTTP method verbs). This value is set between + the 'before_request_body' and 'before_handler' hooks (assuming that + process_request_body is True). + + Deprecated in 3.2, will be removed for 3.3 in favor of + :attr:`request.body.params<cherrypy._cprequest.RequestBody.params>`.""") + + +class ResponseBody(object): + """The body of the HTTP response (the response entity).""" + + def __get__(self, obj, objclass=None): + if obj is None: + # When calling on the class instead of an instance... + return self + else: + return obj._body + + def __set__(self, obj, value): + # Convert the given value to an iterable object. + if isinstance(value, basestring): + # strings get wrapped in a list because iterating over a single + # item list is much faster than iterating over every character + # in a long string. + if value: + value = [value] + else: + # [''] doesn't evaluate to False, so replace it with []. + value = [] + # Don't use isinstance here; io.IOBase which has an ABC takes + # 1000 times as long as, say, isinstance(value, str) + elif hasattr(value, 'read'): + value = file_generator(value) + elif value is None: + value = [] + obj._body = value + + +class Response(object): + """An HTTP Response, including status, headers, and body.""" + + status = "" + """The HTTP Status-Code and Reason-Phrase.""" + + header_list = [] + """ + A list of the HTTP response headers as (name, value) tuples. + In general, you should use response.headers (a dict) instead. This + attribute is generated from response.headers and is not valid until + after the finalize phase.""" + + headers = httputil.HeaderMap() + """ + A dict-like object containing the response headers. Keys are header + names (in Title-Case format); however, you may get and set them in + a case-insensitive manner. That is, headers['Content-Type'] and + headers['content-type'] refer to the same value. Values are header + values (decoded according to :rfc:`2047` if necessary). + + .. seealso:: classes :class:`HeaderMap`, :class:`HeaderElement` + """ + + cookie = SimpleCookie() + """See help(Cookie).""" + + body = ResponseBody() + """The body (entity) of the HTTP response.""" + + time = None + """The value of time.time() when created. Use in HTTP dates.""" + + timeout = 300 + """Seconds after which the response will be aborted.""" + + timed_out = False + """ + Flag to indicate the response should be aborted, because it has + exceeded its timeout.""" + + stream = False + """If False, buffer the response body.""" + + def __init__(self): + self.status = None + self.header_list = None + self._body = [] + self.time = time.time() + + self.headers = httputil.HeaderMap() + # Since we know all our keys are titled strings, we can + # bypass HeaderMap.update and get a big speed boost. + dict.update(self.headers, { + "Content-Type": 'text/html', + "Server": "CherryPy/" + cherrypy.__version__, + "Date": httputil.HTTPDate(self.time), + }) + self.cookie = SimpleCookie() + + def collapse_body(self): + """Collapse self.body to a single string; replace it and return it.""" + if isinstance(self.body, basestring): + return self.body + + newbody = ''.join([chunk for chunk in self.body]) + + self.body = newbody + return newbody + + def finalize(self): + """Transform headers (and cookies) into self.header_list. (Core)""" + try: + code, reason, _ = httputil.valid_status(self.status) + except ValueError: + raise cherrypy.HTTPError(500, sys.exc_info()[1].args[0]) + + headers = self.headers + + self.output_status = ntob(str(code), 'ascii') + ntob(" ") + headers.encode(reason) + + if self.stream: + # The upshot: wsgiserver will chunk the response if + # you pop Content-Length (or set it explicitly to None). + # Note that lib.static sets C-L to the file's st_size. + if dict.get(headers, 'Content-Length') is None: + dict.pop(headers, 'Content-Length', None) + elif code < 200 or code in (204, 205, 304): + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." + dict.pop(headers, 'Content-Length', None) + self.body = ntob("") + else: + # Responses which are not streamed should have a Content-Length, + # but allow user code to set Content-Length if desired. + if dict.get(headers, 'Content-Length') is None: + content = self.collapse_body() + dict.__setitem__(headers, 'Content-Length', len(content)) + + # Transform our header dict into a list of tuples. + self.header_list = h = headers.output() + + cookie = self.cookie.output() + if cookie: + for line in cookie.split("\n"): + if line.endswith("\r"): + # Python 2.4 emits cookies joined by LF but 2.5+ by CRLF. + line = line[:-1] + name, value = line.split(": ", 1) + if isinstance(name, unicodestr): + name = name.encode("ISO-8859-1") + if isinstance(value, unicodestr): + value = headers.encode(value) + h.append((name, value)) + + def check_timeout(self): + """If now > self.time + self.timeout, set self.timed_out. + + This purposefully sets a flag, rather than raising an error, + so that a monitor thread can interrupt the Response thread. + """ + if time.time() > self.time + self.timeout: + self.timed_out = True + + + diff --git a/cherrypy/_cpserver.py b/cherrypy/_cpserver.py new file mode 100755 index 0000000..c1695a6 --- /dev/null +++ b/cherrypy/_cpserver.py @@ -0,0 +1,195 @@ +"""Manage HTTP servers with CherryPy.""" + +import warnings + +import cherrypy +from cherrypy.lib import attributes +from cherrypy._cpcompat import basestring + +# We import * because we want to export check_port +# et al as attributes of this module. +from cherrypy.process.servers import * + + +class Server(ServerAdapter): + """An adapter for an HTTP server. + + You can set attributes (like socket_host and socket_port) + on *this* object (which is probably cherrypy.server), and call + quickstart. For example:: + + cherrypy.server.socket_port = 80 + cherrypy.quickstart() + """ + + socket_port = 8080 + """The TCP port on which to listen for connections.""" + + _socket_host = '127.0.0.1' + def _get_socket_host(self): + return self._socket_host + def _set_socket_host(self, value): + if value == '': + raise ValueError("The empty string ('') is not an allowed value. " + "Use '0.0.0.0' instead to listen on all active " + "interfaces (INADDR_ANY).") + self._socket_host = value + socket_host = property(_get_socket_host, _set_socket_host, + doc="""The hostname or IP address on which to listen for connections. + + Host values may be any IPv4 or IPv6 address, or any valid hostname. + The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if + your hosts file prefers IPv6). The string '0.0.0.0' is a special + IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' + is the similar IN6ADDR_ANY for IPv6. The empty string or None are + not allowed.""") + + socket_file = None + """If given, the name of the UNIX socket to use instead of TCP/IP. + + When this option is not None, the `socket_host` and `socket_port` options + are ignored.""" + + socket_queue_size = 5 + """The 'backlog' argument to socket.listen(); specifies the maximum number + of queued connections (default 5).""" + + socket_timeout = 10 + """The timeout in seconds for accepted connections (default 10).""" + + shutdown_timeout = 5 + """The time to wait for HTTP worker threads to clean up.""" + + protocol_version = 'HTTP/1.1' + """The version string to write in the Status-Line of all HTTP responses, + for example, "HTTP/1.1" (the default). Depending on the HTTP server used, + this should also limit the supported features used in the response.""" + + thread_pool = 10 + """The number of worker threads to start up in the pool.""" + + thread_pool_max = -1 + """The maximum size of the worker-thread pool. Use -1 to indicate no limit.""" + + max_request_header_size = 500 * 1024 + """The maximum number of bytes allowable in the request headers. If exceeded, + the HTTP server should return "413 Request Entity Too Large".""" + + max_request_body_size = 100 * 1024 * 1024 + """The maximum number of bytes allowable in the request body. If exceeded, + the HTTP server should return "413 Request Entity Too Large".""" + + instance = None + """If not None, this should be an HTTP server instance (such as + CPWSGIServer) which cherrypy.server will control. Use this when you need + more control over object instantiation than is available in the various + configuration options.""" + + ssl_context = None + """When using PyOpenSSL, an instance of SSL.Context.""" + + ssl_certificate = None + """The filename of the SSL certificate to use.""" + + ssl_certificate_chain = None + """When using PyOpenSSL, the certificate chain to pass to + Context.load_verify_locations.""" + + ssl_private_key = None + """The filename of the private key to use with SSL.""" + + ssl_module = 'pyopenssl' + """The name of a registered SSL adaptation module to use with the builtin + WSGI server. Builtin options are 'builtin' (to use the SSL library built + into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL + project, which you must install separately). You may also register your + own classes in the wsgiserver.ssl_adapters dict.""" + + nodelay = True + """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" + + wsgi_version = (1, 0) + """The WSGI version tuple to use with the builtin WSGI server. + The provided options are (1, 0) [which includes support for PEP 3333, + which declares it covers WSGI version 1.0.1 but still mandates the + wsgi.version (1, 0)] and ('u', 0), an experimental unicode version. + You may create and register your own experimental versions of the WSGI + protocol by adding custom classes to the wsgiserver.wsgi_gateways dict.""" + + def __init__(self): + self.bus = cherrypy.engine + self.httpserver = None + self.interrupt = None + self.running = False + + def httpserver_from_self(self, httpserver=None): + """Return a (httpserver, bind_addr) pair based on self attributes.""" + if httpserver is None: + httpserver = self.instance + if httpserver is None: + from cherrypy import _cpwsgi_server + httpserver = _cpwsgi_server.CPWSGIServer(self) + if isinstance(httpserver, basestring): + # Is anyone using this? Can I add an arg? + httpserver = attributes(httpserver)(self) + return httpserver, self.bind_addr + + def start(self): + """Start the HTTP server.""" + if not self.httpserver: + self.httpserver, self.bind_addr = self.httpserver_from_self() + ServerAdapter.start(self) + start.priority = 75 + + def _get_bind_addr(self): + if self.socket_file: + return self.socket_file + if self.socket_host is None and self.socket_port is None: + return None + return (self.socket_host, self.socket_port) + def _set_bind_addr(self, value): + if value is None: + self.socket_file = None + self.socket_host = None + self.socket_port = None + elif isinstance(value, basestring): + self.socket_file = value + self.socket_host = None + self.socket_port = None + else: + try: + self.socket_host, self.socket_port = value + self.socket_file = None + except ValueError: + raise ValueError("bind_addr must be a (host, port) tuple " + "(for TCP sockets) or a string (for Unix " + "domain sockets), not %r" % value) + bind_addr = property(_get_bind_addr, _set_bind_addr, + doc='A (host, port) tuple for TCP sockets or a str for Unix domain sockets.') + + def base(self): + """Return the base (scheme://host[:port] or sock file) for this server.""" + if self.socket_file: + return self.socket_file + + host = self.socket_host + if host in ('0.0.0.0', '::'): + # 0.0.0.0 is INADDR_ANY and :: is IN6ADDR_ANY. + # Look up the host name, which should be the + # safest thing to spit out in a URL. + import socket + host = socket.gethostname() + + port = self.socket_port + + if self.ssl_certificate: + scheme = "https" + if port != 443: + host += ":%s" % port + else: + scheme = "http" + if port != 80: + host += ":%s" % port + + return "%s://%s" % (scheme, host) + diff --git a/cherrypy/_cpthreadinglocal.py b/cherrypy/_cpthreadinglocal.py new file mode 100755 index 0000000..34c17ac --- /dev/null +++ b/cherrypy/_cpthreadinglocal.py @@ -0,0 +1,239 @@ +# This is a backport of Python-2.4's threading.local() implementation + +"""Thread-local objects + +(Note that this module provides a Python version of thread + threading.local class. Depending on the version of Python you're + using, there may be a faster one available. You should always import + the local class from threading.) + +Thread-local objects support the management of thread-local data. +If you have data that you want to be local to a thread, simply create +a thread-local object and use its attributes: + + >>> mydata = local() + >>> mydata.number = 42 + >>> mydata.number + 42 + +You can also access the local-object's dictionary: + + >>> mydata.__dict__ + {'number': 42} + >>> mydata.__dict__.setdefault('widgets', []) + [] + >>> mydata.widgets + [] + +What's important about thread-local objects is that their data are +local to a thread. If we access the data in a different thread: + + >>> log = [] + >>> def f(): + ... items = mydata.__dict__.items() + ... items.sort() + ... log.append(items) + ... mydata.number = 11 + ... log.append(mydata.number) + + >>> import threading + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + >>> log + [[], 11] + +we get different data. Furthermore, changes made in the other thread +don't affect data seen in this thread: + + >>> mydata.number + 42 + +Of course, values you get from a local object, including a __dict__ +attribute, are for whatever thread was current at the time the +attribute was read. For that reason, you generally don't want to save +these values across threads, as they apply only to the thread they +came from. + +You can create custom local objects by subclassing the local class: + + >>> class MyLocal(local): + ... number = 2 + ... initialized = False + ... def __init__(self, **kw): + ... if self.initialized: + ... raise SystemError('__init__ called too many times') + ... self.initialized = True + ... self.__dict__.update(kw) + ... def squared(self): + ... return self.number ** 2 + +This can be useful to support default values, methods and +initialization. Note that if you define an __init__ method, it will be +called each time the local object is used in a separate thread. This +is necessary to initialize each thread's dictionary. + +Now if we create a local object: + + >>> mydata = MyLocal(color='red') + +Now we have a default number: + + >>> mydata.number + 2 + +an initial color: + + >>> mydata.color + 'red' + >>> del mydata.color + +And a method that operates on the data: + + >>> mydata.squared() + 4 + +As before, we can access the data in a separate thread: + + >>> log = [] + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + >>> log + [[('color', 'red'), ('initialized', True)], 11] + +without affecting this thread's data: + + >>> mydata.number + 2 + >>> mydata.color + Traceback (most recent call last): + ... + AttributeError: 'MyLocal' object has no attribute 'color' + +Note that subclasses can define slots, but they are not thread +local. They are shared across threads: + + >>> class MyLocal(local): + ... __slots__ = 'number' + + >>> mydata = MyLocal() + >>> mydata.number = 42 + >>> mydata.color = 'red' + +So, the separate thread: + + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + +affects what we see: + + >>> mydata.number + 11 + +>>> del mydata +""" + +# Threading import is at end + +class _localbase(object): + __slots__ = '_local__key', '_local__args', '_local__lock' + + def __new__(cls, *args, **kw): + self = object.__new__(cls) + key = 'thread.local.' + str(id(self)) + object.__setattr__(self, '_local__key', key) + object.__setattr__(self, '_local__args', (args, kw)) + object.__setattr__(self, '_local__lock', RLock()) + + if args or kw and (cls.__init__ is object.__init__): + raise TypeError("Initialization arguments are not supported") + + # We need to create the thread dict in anticipation of + # __init__ being called, to make sure we don't call it + # again ourselves. + dict = object.__getattribute__(self, '__dict__') + currentThread().__dict__[key] = dict + + return self + +def _patch(self): + key = object.__getattribute__(self, '_local__key') + d = currentThread().__dict__.get(key) + if d is None: + d = {} + currentThread().__dict__[key] = d + object.__setattr__(self, '__dict__', d) + + # we have a new instance dict, so call out __init__ if we have + # one + cls = type(self) + if cls.__init__ is not object.__init__: + args, kw = object.__getattribute__(self, '_local__args') + cls.__init__(self, *args, **kw) + else: + object.__setattr__(self, '__dict__', d) + +class local(_localbase): + + def __getattribute__(self, name): + lock = object.__getattribute__(self, '_local__lock') + lock.acquire() + try: + _patch(self) + return object.__getattribute__(self, name) + finally: + lock.release() + + def __setattr__(self, name, value): + lock = object.__getattribute__(self, '_local__lock') + lock.acquire() + try: + _patch(self) + return object.__setattr__(self, name, value) + finally: + lock.release() + + def __delattr__(self, name): + lock = object.__getattribute__(self, '_local__lock') + lock.acquire() + try: + _patch(self) + return object.__delattr__(self, name) + finally: + lock.release() + + + def __del__(): + threading_enumerate = enumerate + __getattribute__ = object.__getattribute__ + + def __del__(self): + key = __getattribute__(self, '_local__key') + + try: + threads = list(threading_enumerate()) + except: + # if enumerate fails, as it seems to do during + # shutdown, we'll skip cleanup under the assumption + # that there is nothing to clean up + return + + for thread in threads: + try: + __dict__ = thread.__dict__ + except AttributeError: + # Thread is dying, rest in peace + continue + + if key in __dict__: + try: + del __dict__[key] + except KeyError: + pass # didn't have anything in this thread + + return __del__ + __del__ = __del__() + +from threading import currentThread, enumerate, RLock diff --git a/cherrypy/_cptools.py b/cherrypy/_cptools.py new file mode 100755 index 0000000..d3eab05 --- /dev/null +++ b/cherrypy/_cptools.py @@ -0,0 +1,510 @@ +"""CherryPy tools. A "tool" is any helper, adapted to CP. + +Tools are usually designed to be used in a variety of ways (although some +may only offer one if they choose): + + Library calls + All tools are callables that can be used wherever needed. + The arguments are straightforward and should be detailed within the + docstring. + + Function decorators + All tools, when called, may be used as decorators which configure + individual CherryPy page handlers (methods on the CherryPy tree). + That is, "@tools.anytool()" should "turn on" the tool via the + decorated function's _cp_config attribute. + + CherryPy config + If a tool exposes a "_setup" callable, it will be called + once per Request (if the feature is "turned on" via config). + +Tools may be implemented as any object with a namespace. The builtins +are generally either modules or instances of the tools.Tool class. +""" + +import sys +import warnings + +import cherrypy + + +def _getargs(func): + """Return the names of all static arguments to the given function.""" + # Use this instead of importing inspect for less mem overhead. + import types + if sys.version_info >= (3, 0): + if isinstance(func, types.MethodType): + func = func.__func__ + co = func.__code__ + else: + if isinstance(func, types.MethodType): + func = func.im_func + co = func.func_code + return co.co_varnames[:co.co_argcount] + + +_attr_error = ("CherryPy Tools cannot be turned on directly. Instead, turn them " + "on via config, or use them as decorators on your page handlers.") + +class Tool(object): + """A registered function for use with CherryPy request-processing hooks. + + help(tool.callable) should give you more information about this Tool. + """ + + namespace = "tools" + + def __init__(self, point, callable, name=None, priority=50): + self._point = point + self.callable = callable + self._name = name + self._priority = priority + self.__doc__ = self.callable.__doc__ + self._setargs() + + def _get_on(self): + raise AttributeError(_attr_error) + def _set_on(self, value): + raise AttributeError(_attr_error) + on = property(_get_on, _set_on) + + def _setargs(self): + """Copy func parameter names to obj attributes.""" + try: + for arg in _getargs(self.callable): + setattr(self, arg, None) + except (TypeError, AttributeError): + if hasattr(self.callable, "__call__"): + for arg in _getargs(self.callable.__call__): + setattr(self, arg, None) + # IronPython 1.0 raises NotImplementedError because + # inspect.getargspec tries to access Python bytecode + # in co_code attribute. + except NotImplementedError: + pass + # IronPython 1B1 may raise IndexError in some cases, + # but if we trap it here it doesn't prevent CP from working. + except IndexError: + pass + + def _merged_args(self, d=None): + """Return a dict of configuration entries for this Tool.""" + if d: + conf = d.copy() + else: + conf = {} + + tm = cherrypy.serving.request.toolmaps[self.namespace] + if self._name in tm: + conf.update(tm[self._name]) + + if "on" in conf: + del conf["on"] + + return conf + + def __call__(self, *args, **kwargs): + """Compile-time decorator (turn on the tool in config). + + For example:: + + @tools.proxy() + def whats_my_base(self): + return cherrypy.request.base + whats_my_base.exposed = True + """ + if args: + raise TypeError("The %r Tool does not accept positional " + "arguments; you must use keyword arguments." + % self._name) + def tool_decorator(f): + if not hasattr(f, "_cp_config"): + f._cp_config = {} + subspace = self.namespace + "." + self._name + "." + f._cp_config[subspace + "on"] = True + for k, v in kwargs.items(): + f._cp_config[subspace + k] = v + return f + return tool_decorator + + def _setup(self): + """Hook this tool into cherrypy.request. + + The standard CherryPy request object will automatically call this + method when the tool is "turned on" in config. + """ + conf = self._merged_args() + p = conf.pop("priority", None) + if p is None: + p = getattr(self.callable, "priority", self._priority) + cherrypy.serving.request.hooks.attach(self._point, self.callable, + priority=p, **conf) + + +class HandlerTool(Tool): + """Tool which is called 'before main', that may skip normal handlers. + + If the tool successfully handles the request (by setting response.body), + if should return True. This will cause CherryPy to skip any 'normal' page + handler. If the tool did not handle the request, it should return False + to tell CherryPy to continue on and call the normal page handler. If the + tool is declared AS a page handler (see the 'handler' method), returning + False will raise NotFound. + """ + + def __init__(self, callable, name=None): + Tool.__init__(self, 'before_handler', callable, name) + + def handler(self, *args, **kwargs): + """Use this tool as a CherryPy page handler. + + For example:: + + class Root: + nav = tools.staticdir.handler(section="/nav", dir="nav", + root=absDir) + """ + def handle_func(*a, **kw): + handled = self.callable(*args, **self._merged_args(kwargs)) + if not handled: + raise cherrypy.NotFound() + return cherrypy.serving.response.body + handle_func.exposed = True + return handle_func + + def _wrapper(self, **kwargs): + if self.callable(**kwargs): + cherrypy.serving.request.handler = None + + def _setup(self): + """Hook this tool into cherrypy.request. + + The standard CherryPy request object will automatically call this + method when the tool is "turned on" in config. + """ + conf = self._merged_args() + p = conf.pop("priority", None) + if p is None: + p = getattr(self.callable, "priority", self._priority) + cherrypy.serving.request.hooks.attach(self._point, self._wrapper, + priority=p, **conf) + + +class HandlerWrapperTool(Tool): + """Tool which wraps request.handler in a provided wrapper function. + + The 'newhandler' arg must be a handler wrapper function that takes a + 'next_handler' argument, plus ``*args`` and ``**kwargs``. Like all + page handler + functions, it must return an iterable for use as cherrypy.response.body. + + For example, to allow your 'inner' page handlers to return dicts + which then get interpolated into a template:: + + def interpolator(next_handler, *args, **kwargs): + filename = cherrypy.request.config.get('template') + cherrypy.response.template = env.get_template(filename) + response_dict = next_handler(*args, **kwargs) + return cherrypy.response.template.render(**response_dict) + cherrypy.tools.jinja = HandlerWrapperTool(interpolator) + """ + + def __init__(self, newhandler, point='before_handler', name=None, priority=50): + self.newhandler = newhandler + self._point = point + self._name = name + self._priority = priority + + def callable(self, debug=False): + innerfunc = cherrypy.serving.request.handler + def wrap(*args, **kwargs): + return self.newhandler(innerfunc, *args, **kwargs) + cherrypy.serving.request.handler = wrap + + +class ErrorTool(Tool): + """Tool which is used to replace the default request.error_response.""" + + def __init__(self, callable, name=None): + Tool.__init__(self, None, callable, name) + + def _wrapper(self): + self.callable(**self._merged_args()) + + def _setup(self): + """Hook this tool into cherrypy.request. + + The standard CherryPy request object will automatically call this + method when the tool is "turned on" in config. + """ + cherrypy.serving.request.error_response = self._wrapper + + +# Builtin tools # + +from cherrypy.lib import cptools, encoding, auth, static, jsontools +from cherrypy.lib import sessions as _sessions, xmlrpc as _xmlrpc +from cherrypy.lib import caching as _caching +from cherrypy.lib import auth_basic, auth_digest + + +class SessionTool(Tool): + """Session Tool for CherryPy. + + sessions.locking + When 'implicit' (the default), the session will be locked for you, + just before running the page handler. + + When 'early', the session will be locked before reading the request + body. This is off by default for safety reasons; for example, + a large upload would block the session, denying an AJAX + progress meter (see http://www.cherrypy.org/ticket/630). + + When 'explicit' (or any other value), you need to call + cherrypy.session.acquire_lock() yourself before using + session data. + """ + + def __init__(self): + # _sessions.init must be bound after headers are read + Tool.__init__(self, 'before_request_body', _sessions.init) + + def _lock_session(self): + cherrypy.serving.session.acquire_lock() + + def _setup(self): + """Hook this tool into cherrypy.request. + + The standard CherryPy request object will automatically call this + method when the tool is "turned on" in config. + """ + hooks = cherrypy.serving.request.hooks + + conf = self._merged_args() + + p = conf.pop("priority", None) + if p is None: + p = getattr(self.callable, "priority", self._priority) + + hooks.attach(self._point, self.callable, priority=p, **conf) + + locking = conf.pop('locking', 'implicit') + if locking == 'implicit': + hooks.attach('before_handler', self._lock_session) + elif locking == 'early': + # Lock before the request body (but after _sessions.init runs!) + hooks.attach('before_request_body', self._lock_session, + priority=60) + else: + # Don't lock + pass + + hooks.attach('before_finalize', _sessions.save) + hooks.attach('on_end_request', _sessions.close) + + def regenerate(self): + """Drop the current session and make a new one (with a new id).""" + sess = cherrypy.serving.session + sess.regenerate() + + # Grab cookie-relevant tool args + conf = dict([(k, v) for k, v in self._merged_args().items() + if k in ('path', 'path_header', 'name', 'timeout', + 'domain', 'secure')]) + _sessions.set_response_cookie(**conf) + + + + +class XMLRPCController(object): + """A Controller (page handler collection) for XML-RPC. + + To use it, have your controllers subclass this base class (it will + turn on the tool for you). + + You can also supply the following optional config entries:: + + tools.xmlrpc.encoding: 'utf-8' + tools.xmlrpc.allow_none: 0 + + XML-RPC is a rather discontinuous layer over HTTP; dispatching to the + appropriate handler must first be performed according to the URL, and + then a second dispatch step must take place according to the RPC method + specified in the request body. It also allows a superfluous "/RPC2" + prefix in the URL, supplies its own handler args in the body, and + requires a 200 OK "Fault" response instead of 404 when the desired + method is not found. + + Therefore, XML-RPC cannot be implemented for CherryPy via a Tool alone. + This Controller acts as the dispatch target for the first half (based + on the URL); it then reads the RPC method from the request body and + does its own second dispatch step based on that method. It also reads + body params, and returns a Fault on error. + + The XMLRPCDispatcher strips any /RPC2 prefix; if you aren't using /RPC2 + in your URL's, you can safely skip turning on the XMLRPCDispatcher. + Otherwise, you need to use declare it in config:: + + request.dispatch: cherrypy.dispatch.XMLRPCDispatcher() + """ + + # Note we're hard-coding this into the 'tools' namespace. We could do + # a huge amount of work to make it relocatable, but the only reason why + # would be if someone actually disabled the default_toolbox. Meh. + _cp_config = {'tools.xmlrpc.on': True} + + def default(self, *vpath, **params): + rpcparams, rpcmethod = _xmlrpc.process_body() + + subhandler = self + for attr in str(rpcmethod).split('.'): + subhandler = getattr(subhandler, attr, None) + + if subhandler and getattr(subhandler, "exposed", False): + body = subhandler(*(vpath + rpcparams), **params) + + else: + # http://www.cherrypy.org/ticket/533 + # if a method is not found, an xmlrpclib.Fault should be returned + # raising an exception here will do that; see + # cherrypy.lib.xmlrpc.on_error + raise Exception('method "%s" is not supported' % attr) + + conf = cherrypy.serving.request.toolmaps['tools'].get("xmlrpc", {}) + _xmlrpc.respond(body, + conf.get('encoding', 'utf-8'), + conf.get('allow_none', 0)) + return cherrypy.serving.response.body + default.exposed = True + + +class SessionAuthTool(HandlerTool): + + def _setargs(self): + for name in dir(cptools.SessionAuth): + if not name.startswith("__"): + setattr(self, name, None) + + +class CachingTool(Tool): + """Caching Tool for CherryPy.""" + + def _wrapper(self, **kwargs): + request = cherrypy.serving.request + if _caching.get(**kwargs): + request.handler = None + else: + if request.cacheable: + # Note the devious technique here of adding hooks on the fly + request.hooks.attach('before_finalize', _caching.tee_output, + priority = 90) + _wrapper.priority = 20 + + def _setup(self): + """Hook caching into cherrypy.request.""" + conf = self._merged_args() + + p = conf.pop("priority", None) + cherrypy.serving.request.hooks.attach('before_handler', self._wrapper, + priority=p, **conf) + + + +class Toolbox(object): + """A collection of Tools. + + This object also functions as a config namespace handler for itself. + Custom toolboxes should be added to each Application's toolboxes dict. + """ + + def __init__(self, namespace): + self.namespace = namespace + + def __setattr__(self, name, value): + # If the Tool._name is None, supply it from the attribute name. + if isinstance(value, Tool): + if value._name is None: + value._name = name + value.namespace = self.namespace + object.__setattr__(self, name, value) + + def __enter__(self): + """Populate request.toolmaps from tools specified in config.""" + cherrypy.serving.request.toolmaps[self.namespace] = map = {} + def populate(k, v): + toolname, arg = k.split(".", 1) + bucket = map.setdefault(toolname, {}) + bucket[arg] = v + return populate + + def __exit__(self, exc_type, exc_val, exc_tb): + """Run tool._setup() for each tool in our toolmap.""" + map = cherrypy.serving.request.toolmaps.get(self.namespace) + if map: + for name, settings in map.items(): + if settings.get("on", False): + tool = getattr(self, name) + tool._setup() + + +class DeprecatedTool(Tool): + + _name = None + warnmsg = "This Tool is deprecated." + + def __init__(self, point, warnmsg=None): + self.point = point + if warnmsg is not None: + self.warnmsg = warnmsg + + def __call__(self, *args, **kwargs): + warnings.warn(self.warnmsg) + def tool_decorator(f): + return f + return tool_decorator + + def _setup(self): + warnings.warn(self.warnmsg) + + +default_toolbox = _d = Toolbox("tools") +_d.session_auth = SessionAuthTool(cptools.session_auth) +_d.allow = Tool('on_start_resource', cptools.allow) +_d.proxy = Tool('before_request_body', cptools.proxy, priority=30) +_d.response_headers = Tool('on_start_resource', cptools.response_headers) +_d.log_tracebacks = Tool('before_error_response', cptools.log_traceback) +_d.log_headers = Tool('before_error_response', cptools.log_request_headers) +_d.log_hooks = Tool('on_end_request', cptools.log_hooks, priority=100) +_d.err_redirect = ErrorTool(cptools.redirect) +_d.etags = Tool('before_finalize', cptools.validate_etags, priority=75) +_d.decode = Tool('before_request_body', encoding.decode) +# the order of encoding, gzip, caching is important +_d.encode = Tool('before_handler', encoding.ResponseEncoder, priority=70) +_d.gzip = Tool('before_finalize', encoding.gzip, priority=80) +_d.staticdir = HandlerTool(static.staticdir) +_d.staticfile = HandlerTool(static.staticfile) +_d.sessions = SessionTool() +_d.xmlrpc = ErrorTool(_xmlrpc.on_error) +_d.caching = CachingTool('before_handler', _caching.get, 'caching') +_d.expires = Tool('before_finalize', _caching.expires) +_d.tidy = DeprecatedTool('before_finalize', + "The tidy tool has been removed from the standard distribution of CherryPy. " + "The most recent version can be found at http://tools.cherrypy.org/browser.") +_d.nsgmls = DeprecatedTool('before_finalize', + "The nsgmls tool has been removed from the standard distribution of CherryPy. " + "The most recent version can be found at http://tools.cherrypy.org/browser.") +_d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) +_d.referer = Tool('before_request_body', cptools.referer) +_d.basic_auth = Tool('on_start_resource', auth.basic_auth) +_d.digest_auth = Tool('on_start_resource', auth.digest_auth) +_d.trailing_slash = Tool('before_handler', cptools.trailing_slash, priority=60) +_d.flatten = Tool('before_finalize', cptools.flatten) +_d.accept = Tool('on_start_resource', cptools.accept) +_d.redirect = Tool('on_start_resource', cptools.redirect) +_d.autovary = Tool('on_start_resource', cptools.autovary, priority=0) +_d.json_in = Tool('before_request_body', jsontools.json_in, priority=30) +_d.json_out = Tool('before_handler', jsontools.json_out, priority=30) +_d.auth_basic = Tool('before_handler', auth_basic.basic_auth, priority=1) +_d.auth_digest = Tool('before_handler', auth_digest.digest_auth, priority=1) + +del _d, cptools, encoding, auth, static diff --git a/cherrypy/_cptree.py b/cherrypy/_cptree.py new file mode 100755 index 0000000..67ce546 --- /dev/null +++ b/cherrypy/_cptree.py @@ -0,0 +1,279 @@ +"""CherryPy Application and Tree objects.""" + +import os +import cherrypy +from cherrypy._cpcompat import ntou +from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools +from cherrypy.lib import httputil + + +class Application(object): + """A CherryPy Application. + + Servers and gateways should not instantiate Request objects directly. + Instead, they should ask an Application object for a request object. + + An instance of this class may also be used as a WSGI callable + (WSGI application object) for itself. + """ + + root = None + """The top-most container of page handlers for this app. Handlers should + be arranged in a hierarchy of attributes, matching the expected URI + hierarchy; the default dispatcher then searches this hierarchy for a + matching handler. When using a dispatcher other than the default, + this value may be None.""" + + config = {} + """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict + of {key: value} pairs.""" + + namespaces = _cpconfig.NamespaceSet() + toolboxes = {'tools': cherrypy.tools} + + log = None + """A LogManager instance. See _cplogging.""" + + wsgiapp = None + """A CPWSGIApp instance. See _cpwsgi.""" + + request_class = _cprequest.Request + response_class = _cprequest.Response + + relative_urls = False + + def __init__(self, root, script_name="", config=None): + self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) + self.root = root + self.script_name = script_name + self.wsgiapp = _cpwsgi.CPWSGIApp(self) + + self.namespaces = self.namespaces.copy() + self.namespaces["log"] = lambda k, v: setattr(self.log, k, v) + self.namespaces["wsgi"] = self.wsgiapp.namespace_handler + + self.config = self.__class__.config.copy() + if config: + self.merge(config) + + def __repr__(self): + return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, + self.root, self.script_name) + + script_name_doc = """The URI "mount point" for this app. A mount point is that portion of + the URI which is constant for all URIs that are serviced by this + application; it does not include scheme, host, or proxy ("virtual host") + portions of the URI. + + For example, if script_name is "/my/cool/app", then the URL + "http://www.example.com/my/cool/app/page1" might be handled by a + "page1" method on the root object. + + The value of script_name MUST NOT end in a slash. If the script_name + refers to the root of the URI, it MUST be an empty string (not "/"). + + If script_name is explicitly set to None, then the script_name will be + provided for each call from request.wsgi_environ['SCRIPT_NAME']. + """ + def _get_script_name(self): + if self._script_name is None: + # None signals that the script name should be pulled from WSGI environ. + return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") + return self._script_name + def _set_script_name(self, value): + if value: + value = value.rstrip("/") + self._script_name = value + script_name = property(fget=_get_script_name, fset=_set_script_name, + doc=script_name_doc) + + def merge(self, config): + """Merge the given config into self.config.""" + _cpconfig.merge(self.config, config) + + # Handle namespaces specified in config. + self.namespaces(self.config.get("/", {})) + + def find_config(self, path, key, default=None): + """Return the most-specific value for key along path, or default.""" + trail = path or "/" + while trail: + nodeconf = self.config.get(trail, {}) + + if key in nodeconf: + return nodeconf[key] + + lastslash = trail.rfind("/") + if lastslash == -1: + break + elif lastslash == 0 and trail != "/": + trail = "/" + else: + trail = trail[:lastslash] + + return default + + def get_serving(self, local, remote, scheme, sproto): + """Create and return a Request and Response object.""" + req = self.request_class(local, remote, scheme, sproto) + req.app = self + + for name, toolbox in self.toolboxes.items(): + req.namespaces[name] = toolbox + + resp = self.response_class() + cherrypy.serving.load(req, resp) + cherrypy.engine.timeout_monitor.acquire() + cherrypy.engine.publish('acquire_thread') + + return req, resp + + def release_serving(self): + """Release the current serving (request and response).""" + req = cherrypy.serving.request + + cherrypy.engine.timeout_monitor.release() + + try: + req.close() + except: + cherrypy.log(traceback=True, severity=40) + + cherrypy.serving.clear() + + def __call__(self, environ, start_response): + return self.wsgiapp(environ, start_response) + + +class Tree(object): + """A registry of CherryPy applications, mounted at diverse points. + + An instance of this class may also be used as a WSGI callable + (WSGI application object), in which case it dispatches to all + mounted apps. + """ + + apps = {} + """ + A dict of the form {script name: application}, where "script name" + is a string declaring the URI mount point (no trailing slash), and + "application" is an instance of cherrypy.Application (or an arbitrary + WSGI callable if you happen to be using a WSGI server).""" + + def __init__(self): + self.apps = {} + + def mount(self, root, script_name="", config=None): + """Mount a new app from a root object, script_name, and config. + + root + An instance of a "controller class" (a collection of page + handler methods) which represents the root of the application. + This may also be an Application instance, or None if using + a dispatcher other than the default. + + script_name + A string containing the "mount point" of the application. + This should start with a slash, and be the path portion of the + URL at which to mount the given root. For example, if root.index() + will handle requests to "http://www.example.com:8080/dept/app1/", + then the script_name argument would be "/dept/app1". + + It MUST NOT end in a slash. If the script_name refers to the + root of the URI, it MUST be an empty string (not "/"). + + config + A file or dict containing application config. + """ + if script_name is None: + raise TypeError( + "The 'script_name' argument may not be None. Application " + "objects may, however, possess a script_name of None (in " + "order to inpect the WSGI environ for SCRIPT_NAME upon each " + "request). You cannot mount such Applications on this Tree; " + "you must pass them to a WSGI server interface directly.") + + # Next line both 1) strips trailing slash and 2) maps "/" -> "". + script_name = script_name.rstrip("/") + + if isinstance(root, Application): + app = root + if script_name != "" and script_name != app.script_name: + raise ValueError("Cannot specify a different script name and " + "pass an Application instance to cherrypy.mount") + script_name = app.script_name + else: + app = Application(root, script_name) + + # If mounted at "", add favicon.ico + if (script_name == "" and root is not None + and not hasattr(root, "favicon_ico")): + favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), + "favicon.ico") + root.favicon_ico = tools.staticfile.handler(favicon) + + if config: + app.merge(config) + + self.apps[script_name] = app + + return app + + def graft(self, wsgi_callable, script_name=""): + """Mount a wsgi callable at the given script_name.""" + # Next line both 1) strips trailing slash and 2) maps "/" -> "". + script_name = script_name.rstrip("/") + self.apps[script_name] = wsgi_callable + + def script_name(self, path=None): + """The script_name of the app at the given path, or None. + + If path is None, cherrypy.request is used. + """ + if path is None: + try: + request = cherrypy.serving.request + path = httputil.urljoin(request.script_name, + request.path_info) + except AttributeError: + return None + + while True: + if path in self.apps: + return path + + if path == "": + return None + + # Move one node up the tree and try again. + path = path[:path.rfind("/")] + + def __call__(self, environ, start_response): + # If you're calling this, then you're probably setting SCRIPT_NAME + # to '' (some WSGI servers always set SCRIPT_NAME to ''). + # Try to look up the app using the full path. + env1x = environ + if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + env1x = _cpwsgi.downgrade_wsgi_ux_to_1x(environ) + path = httputil.urljoin(env1x.get('SCRIPT_NAME', ''), + env1x.get('PATH_INFO', '')) + sn = self.script_name(path or "/") + if sn is None: + start_response('404 Not Found', []) + return [] + + app = self.apps[sn] + + # Correct the SCRIPT_NAME and PATH_INFO environ entries. + environ = environ.copy() + if environ.get(u'wsgi.version') == (u'u', 0): + # Python 2/WSGI u.0: all strings MUST be of type unicode + enc = environ[u'wsgi.url_encoding'] + environ[u'SCRIPT_NAME'] = sn.decode(enc) + environ[u'PATH_INFO'] = path[len(sn.rstrip("/")):].decode(enc) + else: + # Python 2/WSGI 1.x: all strings MUST be of type str + environ['SCRIPT_NAME'] = sn + environ['PATH_INFO'] = path[len(sn.rstrip("/")):] + return app(environ, start_response) + diff --git a/cherrypy/_cpwsgi.py b/cherrypy/_cpwsgi.py new file mode 100755 index 0000000..aa4b763 --- /dev/null +++ b/cherrypy/_cpwsgi.py @@ -0,0 +1,347 @@ +"""WSGI interface (see PEP 333 and 3333). + +Note that WSGI environ keys and values are 'native strings'; that is, +whatever the type of "" is. For Python 2, that's a byte string; for Python 3, +it's a unicode string. But PEP 3333 says: "even if Python's str type is +actually Unicode "under the hood", the content of native strings must +still be translatable to bytes via the Latin-1 encoding!" +""" + +import sys as _sys + +import cherrypy as _cherrypy +from cherrypy._cpcompat import BytesIO +from cherrypy import _cperror +from cherrypy.lib import httputil + + +def downgrade_wsgi_ux_to_1x(environ): + """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ.""" + env1x = {} + + url_encoding = environ[u'wsgi.url_encoding'] + for k, v in environ.items(): + if k in [u'PATH_INFO', u'SCRIPT_NAME', u'QUERY_STRING']: + v = v.encode(url_encoding) + elif isinstance(v, unicode): + v = v.encode('ISO-8859-1') + env1x[k.encode('ISO-8859-1')] = v + + return env1x + + +class VirtualHost(object): + """Select a different WSGI application based on the Host header. + + This can be useful when running multiple sites within one CP server. + It allows several domains to point to different applications. For example:: + + root = Root() + RootApp = cherrypy.Application(root) + Domain2App = cherrypy.Application(root) + SecureApp = cherrypy.Application(Secure()) + + vhost = cherrypy._cpwsgi.VirtualHost(RootApp, + domains={'www.domain2.example': Domain2App, + 'www.domain2.example:443': SecureApp, + }) + + cherrypy.tree.graft(vhost) + """ + default = None + """Required. The default WSGI application.""" + + use_x_forwarded_host = True + """If True (the default), any "X-Forwarded-Host" + request header will be used instead of the "Host" header. This + is commonly added by HTTP servers (such as Apache) when proxying.""" + + domains = {} + """A dict of {host header value: application} pairs. + The incoming "Host" request header is looked up in this dict, + and, if a match is found, the corresponding WSGI application + will be called instead of the default. Note that you often need + separate entries for "example.com" and "www.example.com". + In addition, "Host" headers may contain the port number. + """ + + def __init__(self, default, domains=None, use_x_forwarded_host=True): + self.default = default + self.domains = domains or {} + self.use_x_forwarded_host = use_x_forwarded_host + + def __call__(self, environ, start_response): + domain = environ.get('HTTP_HOST', '') + if self.use_x_forwarded_host: + domain = environ.get("HTTP_X_FORWARDED_HOST", domain) + + nextapp = self.domains.get(domain) + if nextapp is None: + nextapp = self.default + return nextapp(environ, start_response) + + +class InternalRedirector(object): + """WSGI middleware that handles raised cherrypy.InternalRedirect.""" + + def __init__(self, nextapp, recursive=False): + self.nextapp = nextapp + self.recursive = recursive + + def __call__(self, environ, start_response): + redirections = [] + while True: + environ = environ.copy() + try: + return self.nextapp(environ, start_response) + except _cherrypy.InternalRedirect, ir: + sn = environ.get('SCRIPT_NAME', '') + path = environ.get('PATH_INFO', '') + qs = environ.get('QUERY_STRING', '') + + # Add the *previous* path_info + qs to redirections. + old_uri = sn + path + if qs: + old_uri += "?" + qs + redirections.append(old_uri) + + if not self.recursive: + # Check to see if the new URI has been redirected to already + new_uri = sn + ir.path + if ir.query_string: + new_uri += "?" + ir.query_string + if new_uri in redirections: + ir.request.close() + raise RuntimeError("InternalRedirector visited the " + "same URL twice: %r" % new_uri) + + # Munge the environment and try again. + environ['REQUEST_METHOD'] = "GET" + environ['PATH_INFO'] = ir.path + environ['QUERY_STRING'] = ir.query_string + environ['wsgi.input'] = BytesIO() + environ['CONTENT_LENGTH'] = "0" + environ['cherrypy.previous_request'] = ir.request + + +class ExceptionTrapper(object): + """WSGI middleware that traps exceptions.""" + + def __init__(self, nextapp, throws=(KeyboardInterrupt, SystemExit)): + self.nextapp = nextapp + self.throws = throws + + def __call__(self, environ, start_response): + return _TrappedResponse(self.nextapp, environ, start_response, self.throws) + + +class _TrappedResponse(object): + + response = iter([]) + + def __init__(self, nextapp, environ, start_response, throws): + self.nextapp = nextapp + self.environ = environ + self.start_response = start_response + self.throws = throws + self.started_response = False + self.response = self.trap(self.nextapp, self.environ, self.start_response) + self.iter_response = iter(self.response) + + def __iter__(self): + self.started_response = True + return self + + def next(self): + return self.trap(self.iter_response.next) + + def close(self): + if hasattr(self.response, 'close'): + self.response.close() + + def trap(self, func, *args, **kwargs): + try: + return func(*args, **kwargs) + except self.throws: + raise + except StopIteration: + raise + except: + tb = _cperror.format_exc() + #print('trapped (started %s):' % self.started_response, tb) + _cherrypy.log(tb, severity=40) + if not _cherrypy.request.show_tracebacks: + tb = "" + s, h, b = _cperror.bare_error(tb) + if self.started_response: + # Empty our iterable (so future calls raise StopIteration) + self.iter_response = iter([]) + else: + self.iter_response = iter(b) + + try: + self.start_response(s, h, _sys.exc_info()) + except: + # "The application must not trap any exceptions raised by + # start_response, if it called start_response with exc_info. + # Instead, it should allow such exceptions to propagate + # back to the server or gateway." + # But we still log and call close() to clean up ourselves. + _cherrypy.log(traceback=True, severity=40) + raise + + if self.started_response: + return "".join(b) + else: + return b + + +# WSGI-to-CP Adapter # + + +class AppResponse(object): + """WSGI response iterable for CherryPy applications.""" + + def __init__(self, environ, start_response, cpapp): + if environ.get(u'wsgi.version') == (u'u', 0): + environ = downgrade_wsgi_ux_to_1x(environ) + self.environ = environ + self.cpapp = cpapp + try: + self.run() + except: + self.close() + raise + r = _cherrypy.serving.response + self.iter_response = iter(r.body) + self.write = start_response(r.output_status, r.header_list) + + def __iter__(self): + return self + + def next(self): + return self.iter_response.next() + + def close(self): + """Close and de-reference the current request and response. (Core)""" + self.cpapp.release_serving() + + def run(self): + """Create a Request object using environ.""" + env = self.environ.get + + local = httputil.Host('', int(env('SERVER_PORT', 80)), + env('SERVER_NAME', '')) + remote = httputil.Host(env('REMOTE_ADDR', ''), + int(env('REMOTE_PORT', -1) or -1), + env('REMOTE_HOST', '')) + scheme = env('wsgi.url_scheme') + sproto = env('ACTUAL_SERVER_PROTOCOL', "HTTP/1.1") + request, resp = self.cpapp.get_serving(local, remote, scheme, sproto) + + # LOGON_USER is served by IIS, and is the name of the + # user after having been mapped to a local account. + # Both IIS and Apache set REMOTE_USER, when possible. + request.login = env('LOGON_USER') or env('REMOTE_USER') or None + request.multithread = self.environ['wsgi.multithread'] + request.multiprocess = self.environ['wsgi.multiprocess'] + request.wsgi_environ = self.environ + request.prev = env('cherrypy.previous_request', None) + + meth = self.environ['REQUEST_METHOD'] + + path = httputil.urljoin(self.environ.get('SCRIPT_NAME', ''), + self.environ.get('PATH_INFO', '')) + qs = self.environ.get('QUERY_STRING', '') + rproto = self.environ.get('SERVER_PROTOCOL') + headers = self.translate_headers(self.environ) + rfile = self.environ['wsgi.input'] + request.run(meth, path, qs, rproto, headers, rfile) + + headerNames = {'HTTP_CGI_AUTHORIZATION': 'Authorization', + 'CONTENT_LENGTH': 'Content-Length', + 'CONTENT_TYPE': 'Content-Type', + 'REMOTE_HOST': 'Remote-Host', + 'REMOTE_ADDR': 'Remote-Addr', + } + + def translate_headers(self, environ): + """Translate CGI-environ header names to HTTP header names.""" + for cgiName in environ: + # We assume all incoming header keys are uppercase already. + if cgiName in self.headerNames: + yield self.headerNames[cgiName], environ[cgiName] + elif cgiName[:5] == "HTTP_": + # Hackish attempt at recovering original header names. + translatedHeader = cgiName[5:].replace("_", "-") + yield translatedHeader, environ[cgiName] + + +class CPWSGIApp(object): + """A WSGI application object for a CherryPy Application.""" + + pipeline = [('ExceptionTrapper', ExceptionTrapper), + ('InternalRedirector', InternalRedirector), + ] + """A list of (name, wsgiapp) pairs. Each 'wsgiapp' MUST be a + constructor that takes an initial, positional 'nextapp' argument, + plus optional keyword arguments, and returns a WSGI application + (that takes environ and start_response arguments). The 'name' can + be any you choose, and will correspond to keys in self.config.""" + + head = None + """Rather than nest all apps in the pipeline on each call, it's only + done the first time, and the result is memoized into self.head. Set + this to None again if you change self.pipeline after calling self.""" + + config = {} + """A dict whose keys match names listed in the pipeline. Each + value is a further dict which will be passed to the corresponding + named WSGI callable (from the pipeline) as keyword arguments.""" + + response_class = AppResponse + """The class to instantiate and return as the next app in the WSGI chain.""" + + def __init__(self, cpapp, pipeline=None): + self.cpapp = cpapp + self.pipeline = self.pipeline[:] + if pipeline: + self.pipeline.extend(pipeline) + self.config = self.config.copy() + + def tail(self, environ, start_response): + """WSGI application callable for the actual CherryPy application. + + You probably shouldn't call this; call self.__call__ instead, + so that any WSGI middleware in self.pipeline can run first. + """ + return self.response_class(environ, start_response, self.cpapp) + + def __call__(self, environ, start_response): + head = self.head + if head is None: + # Create and nest the WSGI apps in our pipeline (in reverse order). + # Then memoize the result in self.head. + head = self.tail + for name, callable in self.pipeline[::-1]: + conf = self.config.get(name, {}) + head = callable(head, **conf) + self.head = head + return head(environ, start_response) + + def namespace_handler(self, k, v): + """Config handler for the 'wsgi' namespace.""" + if k == "pipeline": + # Note this allows multiple 'wsgi.pipeline' config entries + # (but each entry will be processed in a 'random' order). + # It should also allow developers to set default middleware + # in code (passed to self.__init__) that deployers can add to + # (but not remove) via config. + self.pipeline.extend(v) + elif k == "response_class": + self.response_class = v + else: + name, arg = k.split(".", 1) + bucket = self.config.setdefault(name, {}) + bucket[arg] = v + diff --git a/cherrypy/_cpwsgi_server.py b/cherrypy/_cpwsgi_server.py new file mode 100755 index 0000000..49fd5a1 --- /dev/null +++ b/cherrypy/_cpwsgi_server.py @@ -0,0 +1,54 @@ +"""WSGI server interface (see PEP 333). This adds some CP-specific bits to +the framework-agnostic wsgiserver package. +""" +import sys + +import cherrypy +from cherrypy import wsgiserver + + +class CPWSGIServer(wsgiserver.CherryPyWSGIServer): + """Wrapper for wsgiserver.CherryPyWSGIServer. + + wsgiserver has been designed to not reference CherryPy in any way, + so that it can be used in other frameworks and applications. Therefore, + we wrap it here, so we can set our own mount points from cherrypy.tree + and apply some attributes from config -> cherrypy.server -> wsgiserver. + """ + + def __init__(self, server_adapter=cherrypy.server): + self.server_adapter = server_adapter + self.max_request_header_size = self.server_adapter.max_request_header_size or 0 + self.max_request_body_size = self.server_adapter.max_request_body_size or 0 + + server_name = (self.server_adapter.socket_host or + self.server_adapter.socket_file or + None) + + self.wsgi_version = self.server_adapter.wsgi_version + s = wsgiserver.CherryPyWSGIServer + s.__init__(self, server_adapter.bind_addr, cherrypy.tree, + self.server_adapter.thread_pool, + server_name, + max = self.server_adapter.thread_pool_max, + request_queue_size = self.server_adapter.socket_queue_size, + timeout = self.server_adapter.socket_timeout, + shutdown_timeout = self.server_adapter.shutdown_timeout, + ) + self.protocol = self.server_adapter.protocol_version + self.nodelay = self.server_adapter.nodelay + + ssl_module = self.server_adapter.ssl_module or 'pyopenssl' + if self.server_adapter.ssl_context: + adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + self.ssl_adapter = adapter_class( + self.server_adapter.ssl_certificate, + self.server_adapter.ssl_private_key, + self.server_adapter.ssl_certificate_chain) + self.ssl_adapter.context = self.server_adapter.ssl_context + elif self.server_adapter.ssl_certificate: + adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + self.ssl_adapter = adapter_class( + self.server_adapter.ssl_certificate, + self.server_adapter.ssl_private_key, + self.server_adapter.ssl_certificate_chain) diff --git a/cherrypy/cherryd b/cherrypy/cherryd new file mode 100644 index 0000000..adb2a02 --- /dev/null +++ b/cherrypy/cherryd @@ -0,0 +1,109 @@ +#! /usr/bin/env python +"""The CherryPy daemon.""" + +import sys + +import cherrypy +from cherrypy.process import plugins, servers +from cherrypy import Application + +def start(configfiles=None, daemonize=False, environment=None, + fastcgi=False, scgi=False, pidfile=None, imports=None, + cgi=False): + """Subscribe all engine plugins and start the engine.""" + sys.path = [''] + sys.path + for i in imports or []: + exec("import %s" % i) + + for c in configfiles or []: + cherrypy.config.update(c) + # If there's only one app mounted, merge config into it. + if len(cherrypy.tree.apps) == 1: + for app in cherrypy.tree.apps.values(): + if isinstance(app, Application): + app.merge(c) + + engine = cherrypy.engine + + if environment is not None: + cherrypy.config.update({'environment': environment}) + + # Only daemonize if asked to. + if daemonize: + # Don't print anything to stdout/sterr. + cherrypy.config.update({'log.screen': False}) + plugins.Daemonizer(engine).subscribe() + + if pidfile: + plugins.PIDFile(engine, pidfile).subscribe() + + if hasattr(engine, "signal_handler"): + engine.signal_handler.subscribe() + if hasattr(engine, "console_control_handler"): + engine.console_control_handler.subscribe() + + if (fastcgi and (scgi or cgi)) or (scgi and cgi): + cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " + "scgi options.", 'ENGINE') + sys.exit(1) + elif fastcgi or scgi or cgi: + # Turn off autoreload when using *cgi. + cherrypy.config.update({'engine.autoreload_on': False}) + # Turn off the default HTTP server (which is subscribed by default). + cherrypy.server.unsubscribe() + + addr = cherrypy.server.bind_addr + if fastcgi: + f = servers.FlupFCGIServer(application=cherrypy.tree, + bindAddress=addr) + elif scgi: + f = servers.FlupSCGIServer(application=cherrypy.tree, + bindAddress=addr) + else: + f = servers.FlupCGIServer(application=cherrypy.tree, + bindAddress=addr) + s = servers.ServerAdapter(engine, httpserver=f, bind_addr=addr) + s.subscribe() + + # Always start the engine; this will start all other services + try: + engine.start() + except: + # Assume the error has been logged already via bus.log. + sys.exit(1) + else: + engine.block() + + +if __name__ == '__main__': + from optparse import OptionParser + + p = OptionParser() + p.add_option('-c', '--config', action="append", dest='config', + help="specify config file(s)") + p.add_option('-d', action="store_true", dest='daemonize', + help="run the server as a daemon") + p.add_option('-e', '--environment', dest='environment', default=None, + help="apply the given config environment") + p.add_option('-f', action="store_true", dest='fastcgi', + help="start a fastcgi server instead of the default HTTP server") + p.add_option('-s', action="store_true", dest='scgi', + help="start a scgi server instead of the default HTTP server") + p.add_option('-x', action="store_true", dest='cgi', + help="start a cgi server instead of the default HTTP server") + p.add_option('-i', '--import', action="append", dest='imports', + help="specify modules to import") + p.add_option('-p', '--pidfile', dest='pidfile', default=None, + help="store the process id in the given file") + p.add_option('-P', '--Path', action="append", dest='Path', + help="add the given paths to sys.path") + options, args = p.parse_args() + + if options.Path: + for p in options.Path: + sys.path.insert(0, p) + + start(options.config, options.daemonize, + options.environment, options.fastcgi, options.scgi, + options.pidfile, options.imports, options.cgi) + diff --git a/cherrypy/favicon.ico b/cherrypy/favicon.ico Binary files differnew file mode 100644 index 0000000..f0d7e61 --- /dev/null +++ b/cherrypy/favicon.ico diff --git a/cherrypy/lib/__init__.py b/cherrypy/lib/__init__.py new file mode 100755 index 0000000..611350c --- /dev/null +++ b/cherrypy/lib/__init__.py @@ -0,0 +1,45 @@ +"""CherryPy Library""" + +# Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 +from cherrypy.lib.reprconf import _Builder, unrepr, modules, attributes + +class file_generator(object): + """Yield the given input (a file object) in chunks (default 64k). (Core)""" + + def __init__(self, input, chunkSize=65536): + self.input = input + self.chunkSize = chunkSize + + def __iter__(self): + return self + + def __next__(self): + chunk = self.input.read(self.chunkSize) + if chunk: + return chunk + else: + if hasattr(self.input, 'close'): + self.input.close() + raise StopIteration() + next = __next__ + +def file_generator_limited(fileobj, count, chunk_size=65536): + """Yield the given file object in chunks, stopping after `count` + bytes has been emitted. Default chunk size is 64kB. (Core) + """ + remaining = count + while remaining > 0: + chunk = fileobj.read(min(chunk_size, remaining)) + chunklen = len(chunk) + if chunklen == 0: + return + remaining -= chunklen + yield chunk + +def set_vary_header(response, header_name): + "Add a Vary header to a response" + varies = response.headers.get("Vary", "") + varies = [x.strip() for x in varies.split(",") if x.strip()] + if header_name not in varies: + varies.append(header_name) + response.headers['Vary'] = ", ".join(varies) diff --git a/cherrypy/lib/auth.py b/cherrypy/lib/auth.py new file mode 100755 index 0000000..7d2f6dc --- /dev/null +++ b/cherrypy/lib/auth.py @@ -0,0 +1,87 @@ +import cherrypy +from cherrypy.lib import httpauth + + +def check_auth(users, encrypt=None, realm=None): + """If an authorization header contains credentials, return True, else False.""" + request = cherrypy.serving.request + if 'authorization' in request.headers: + # make sure the provided credentials are correctly set + ah = httpauth.parseAuthorization(request.headers['authorization']) + if ah is None: + raise cherrypy.HTTPError(400, 'Bad Request') + + if not encrypt: + encrypt = httpauth.DIGEST_AUTH_ENCODERS[httpauth.MD5] + + if hasattr(users, '__call__'): + try: + # backward compatibility + users = users() # expect it to return a dictionary + + if not isinstance(users, dict): + raise ValueError("Authentication users must be a dictionary") + + # fetch the user password + password = users.get(ah["username"], None) + except TypeError: + # returns a password (encrypted or clear text) + password = users(ah["username"]) + else: + if not isinstance(users, dict): + raise ValueError("Authentication users must be a dictionary") + + # fetch the user password + password = users.get(ah["username"], None) + + # validate the authorization by re-computing it here + # and compare it with what the user-agent provided + if httpauth.checkResponse(ah, password, method=request.method, + encrypt=encrypt, realm=realm): + request.login = ah["username"] + return True + + request.login = False + return False + +def basic_auth(realm, users, encrypt=None, debug=False): + """If auth fails, raise 401 with a basic authentication header. + + realm + A string containing the authentication realm. + + users + A dict of the form: {username: password} or a callable returning a dict. + + encrypt + callable used to encrypt the password returned from the user-agent. + if None it defaults to a md5 encryption. + + """ + if check_auth(users, encrypt): + if debug: + cherrypy.log('Auth successful', 'TOOLS.BASIC_AUTH') + return + + # inform the user-agent this path is protected + cherrypy.serving.response.headers['www-authenticate'] = httpauth.basicAuth(realm) + + raise cherrypy.HTTPError(401, "You are not authorized to access that resource") + +def digest_auth(realm, users, debug=False): + """If auth fails, raise 401 with a digest authentication header. + + realm + A string containing the authentication realm. + users + A dict of the form: {username: password} or a callable returning a dict. + """ + if check_auth(users, realm=realm): + if debug: + cherrypy.log('Auth successful', 'TOOLS.DIGEST_AUTH') + return + + # inform the user-agent this path is protected + cherrypy.serving.response.headers['www-authenticate'] = httpauth.digestAuth(realm) + + raise cherrypy.HTTPError(401, "You are not authorized to access that resource") diff --git a/cherrypy/lib/auth_basic.py b/cherrypy/lib/auth_basic.py new file mode 100755 index 0000000..2c05e01 --- /dev/null +++ b/cherrypy/lib/auth_basic.py @@ -0,0 +1,87 @@ +# This file is part of CherryPy <http://www.cherrypy.org/> +# -*- coding: utf-8 -*- +# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 + +__doc__ = """This module provides a CherryPy 3.x tool which implements +the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. + +Example usage, using the built-in checkpassword_dict function which uses a dict +as the credentials store:: + + userpassdict = {'bird' : 'bebop', 'ornette' : 'wayout'} + checkpassword = cherrypy.lib.auth_basic.checkpassword_dict(userpassdict) + basic_auth = {'tools.auth_basic.on': True, + 'tools.auth_basic.realm': 'earth', + 'tools.auth_basic.checkpassword': checkpassword, + } + app_config = { '/' : basic_auth } + +""" + +__author__ = 'visteya' +__date__ = 'April 2009' + +import binascii +from cherrypy._cpcompat import base64_decode +import cherrypy + + +def checkpassword_dict(user_password_dict): + """Returns a checkpassword function which checks credentials + against a dictionary of the form: {username : password}. + + If you want a simple dictionary-based authentication scheme, use + checkpassword_dict(my_credentials_dict) as the value for the + checkpassword argument to basic_auth(). + """ + def checkpassword(realm, user, password): + p = user_password_dict.get(user) + return p and p == password or False + + return checkpassword + + +def basic_auth(realm, checkpassword, debug=False): + """A CherryPy tool which hooks at before_handler to perform + HTTP Basic Access Authentication, as specified in :rfc:`2617`. + + If the request has an 'authorization' header with a 'Basic' scheme, this + tool attempts to authenticate the credentials supplied in that header. If + the request has no 'authorization' header, or if it does but the scheme is + not 'Basic', or if authentication fails, the tool sends a 401 response with + a 'WWW-Authenticate' Basic header. + + realm + A string containing the authentication realm. + + checkpassword + A callable which checks the authentication credentials. + Its signature is checkpassword(realm, username, password). where + username and password are the values obtained from the request's + 'authorization' header. If authentication succeeds, checkpassword + returns True, else it returns False. + + """ + + if '"' in realm: + raise ValueError('Realm cannot contain the " (quote) character.') + request = cherrypy.serving.request + + auth_header = request.headers.get('authorization') + if auth_header is not None: + try: + scheme, params = auth_header.split(' ', 1) + if scheme.lower() == 'basic': + username, password = base64_decode(params).split(':', 1) + if checkpassword(realm, username, password): + if debug: + cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') + request.login = username + return # successful authentication + except (ValueError, binascii.Error): # split() error, base64.decodestring() error + raise cherrypy.HTTPError(400, 'Bad Request') + + # Respond with 401 status and a WWW-Authenticate header + cherrypy.serving.response.headers['www-authenticate'] = 'Basic realm="%s"' % realm + raise cherrypy.HTTPError(401, "You are not authorized to access that resource") + diff --git a/cherrypy/lib/auth_digest.py b/cherrypy/lib/auth_digest.py new file mode 100755 index 0000000..67578e0 --- /dev/null +++ b/cherrypy/lib/auth_digest.py @@ -0,0 +1,365 @@ +# This file is part of CherryPy <http://www.cherrypy.org/> +# -*- coding: utf-8 -*- +# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 + +__doc__ = """An implementation of the server-side of HTTP Digest Access +Authentication, which is described in :rfc:`2617`. + +Example usage, using the built-in get_ha1_dict_plain function which uses a dict +of plaintext passwords as the credentials store:: + + userpassdict = {'alice' : '4x5istwelve'} + get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict) + digest_auth = {'tools.auth_digest.on': True, + 'tools.auth_digest.realm': 'wonderland', + 'tools.auth_digest.get_ha1': get_ha1, + 'tools.auth_digest.key': 'a565c27146791cfb', + } + app_config = { '/' : digest_auth } +""" + +__author__ = 'visteya' +__date__ = 'April 2009' + + +import time +from cherrypy._cpcompat import parse_http_list, parse_keqv_list + +import cherrypy +from cherrypy._cpcompat import md5, ntob +md5_hex = lambda s: md5(ntob(s)).hexdigest() + +qop_auth = 'auth' +qop_auth_int = 'auth-int' +valid_qops = (qop_auth, qop_auth_int) + +valid_algorithms = ('MD5', 'MD5-sess') + + +def TRACE(msg): + cherrypy.log(msg, context='TOOLS.AUTH_DIGEST') + +# Three helper functions for users of the tool, providing three variants +# of get_ha1() functions for three different kinds of credential stores. +def get_ha1_dict_plain(user_password_dict): + """Returns a get_ha1 function which obtains a plaintext password from a + dictionary of the form: {username : password}. + + If you want a simple dictionary-based authentication scheme, with plaintext + passwords, use get_ha1_dict_plain(my_userpass_dict) as the value for the + get_ha1 argument to digest_auth(). + """ + def get_ha1(realm, username): + password = user_password_dict.get(username) + if password: + return md5_hex('%s:%s:%s' % (username, realm, password)) + return None + + return get_ha1 + +def get_ha1_dict(user_ha1_dict): + """Returns a get_ha1 function which obtains a HA1 password hash from a + dictionary of the form: {username : HA1}. + + If you want a dictionary-based authentication scheme, but with + pre-computed HA1 hashes instead of plain-text passwords, use + get_ha1_dict(my_userha1_dict) as the value for the get_ha1 + argument to digest_auth(). + """ + def get_ha1(realm, username): + return user_ha1_dict.get(user) + + return get_ha1 + +def get_ha1_file_htdigest(filename): + """Returns a get_ha1 function which obtains a HA1 password hash from a + flat file with lines of the same format as that produced by the Apache + htdigest utility. For example, for realm 'wonderland', username 'alice', + and password '4x5istwelve', the htdigest line would be:: + + alice:wonderland:3238cdfe91a8b2ed8e39646921a02d4c + + If you want to use an Apache htdigest file as the credentials store, + then use get_ha1_file_htdigest(my_htdigest_file) as the value for the + get_ha1 argument to digest_auth(). It is recommended that the filename + argument be an absolute path, to avoid problems. + """ + def get_ha1(realm, username): + result = None + f = open(filename, 'r') + for line in f: + u, r, ha1 = line.rstrip().split(':') + if u == username and r == realm: + result = ha1 + break + f.close() + return result + + return get_ha1 + + +def synthesize_nonce(s, key, timestamp=None): + """Synthesize a nonce value which resists spoofing and can be checked for staleness. + Returns a string suitable as the value for 'nonce' in the www-authenticate header. + + s + A string related to the resource, such as the hostname of the server. + + key + A secret string known only to the server. + + timestamp + An integer seconds-since-the-epoch timestamp + + """ + if timestamp is None: + timestamp = int(time.time()) + h = md5_hex('%s:%s:%s' % (timestamp, s, key)) + nonce = '%s:%s' % (timestamp, h) + return nonce + + +def H(s): + """The hash function H""" + return md5_hex(s) + + +class HttpDigestAuthorization (object): + """Class to parse a Digest Authorization header and perform re-calculation + of the digest. + """ + + def errmsg(self, s): + return 'Digest Authorization header: %s' % s + + def __init__(self, auth_header, http_method, debug=False): + self.http_method = http_method + self.debug = debug + scheme, params = auth_header.split(" ", 1) + self.scheme = scheme.lower() + if self.scheme != 'digest': + raise ValueError('Authorization scheme is not "Digest"') + + self.auth_header = auth_header + + # make a dict of the params + items = parse_http_list(params) + paramsd = parse_keqv_list(items) + + self.realm = paramsd.get('realm') + self.username = paramsd.get('username') + self.nonce = paramsd.get('nonce') + self.uri = paramsd.get('uri') + self.method = paramsd.get('method') + self.response = paramsd.get('response') # the response digest + self.algorithm = paramsd.get('algorithm', 'MD5') + self.cnonce = paramsd.get('cnonce') + self.opaque = paramsd.get('opaque') + self.qop = paramsd.get('qop') # qop + self.nc = paramsd.get('nc') # nonce count + + # perform some correctness checks + if self.algorithm not in valid_algorithms: + raise ValueError(self.errmsg("Unsupported value for algorithm: '%s'" % self.algorithm)) + + has_reqd = self.username and \ + self.realm and \ + self.nonce and \ + self.uri and \ + self.response + if not has_reqd: + raise ValueError(self.errmsg("Not all required parameters are present.")) + + if self.qop: + if self.qop not in valid_qops: + raise ValueError(self.errmsg("Unsupported value for qop: '%s'" % self.qop)) + if not (self.cnonce and self.nc): + raise ValueError(self.errmsg("If qop is sent then cnonce and nc MUST be present")) + else: + if self.cnonce or self.nc: + raise ValueError(self.errmsg("If qop is not sent, neither cnonce nor nc can be present")) + + + def __str__(self): + return 'authorization : %s' % self.auth_header + + def validate_nonce(self, s, key): + """Validate the nonce. + Returns True if nonce was generated by synthesize_nonce() and the timestamp + is not spoofed, else returns False. + + s + A string related to the resource, such as the hostname of the server. + + key + A secret string known only to the server. + + Both s and key must be the same values which were used to synthesize the nonce + we are trying to validate. + """ + try: + timestamp, hashpart = self.nonce.split(':', 1) + s_timestamp, s_hashpart = synthesize_nonce(s, key, timestamp).split(':', 1) + is_valid = s_hashpart == hashpart + if self.debug: + TRACE('validate_nonce: %s' % is_valid) + return is_valid + except ValueError: # split() error + pass + return False + + + def is_nonce_stale(self, max_age_seconds=600): + """Returns True if a validated nonce is stale. The nonce contains a + timestamp in plaintext and also a secure hash of the timestamp. You should + first validate the nonce to ensure the plaintext timestamp is not spoofed. + """ + try: + timestamp, hashpart = self.nonce.split(':', 1) + if int(timestamp) + max_age_seconds > int(time.time()): + return False + except ValueError: # int() error + pass + if self.debug: + TRACE("nonce is stale") + return True + + + def HA2(self, entity_body=''): + """Returns the H(A2) string. See :rfc:`2617` section 3.2.2.3.""" + # RFC 2617 3.2.2.3 + # If the "qop" directive's value is "auth" or is unspecified, then A2 is: + # A2 = method ":" digest-uri-value + # + # If the "qop" value is "auth-int", then A2 is: + # A2 = method ":" digest-uri-value ":" H(entity-body) + if self.qop is None or self.qop == "auth": + a2 = '%s:%s' % (self.http_method, self.uri) + elif self.qop == "auth-int": + a2 = "%s:%s:%s" % (self.http_method, self.uri, H(entity_body)) + else: + # in theory, this should never happen, since I validate qop in __init__() + raise ValueError(self.errmsg("Unrecognized value for qop!")) + return H(a2) + + + def request_digest(self, ha1, entity_body=''): + """Calculates the Request-Digest. See :rfc:`2617` section 3.2.2.1. + + ha1 + The HA1 string obtained from the credentials store. + + entity_body + If 'qop' is set to 'auth-int', then A2 includes a hash + of the "entity body". The entity body is the part of the + message which follows the HTTP headers. See :rfc:`2617` section + 4.3. This refers to the entity the user agent sent in the request which + has the Authorization header. Typically GET requests don't have an entity, + and POST requests do. + + """ + ha2 = self.HA2(entity_body) + # Request-Digest -- RFC 2617 3.2.2.1 + if self.qop: + req = "%s:%s:%s:%s:%s" % (self.nonce, self.nc, self.cnonce, self.qop, ha2) + else: + req = "%s:%s" % (self.nonce, ha2) + + # RFC 2617 3.2.2.2 + # + # If the "algorithm" directive's value is "MD5" or is unspecified, then A1 is: + # A1 = unq(username-value) ":" unq(realm-value) ":" passwd + # + # If the "algorithm" directive's value is "MD5-sess", then A1 is + # calculated only once - on the first request by the client following + # receipt of a WWW-Authenticate challenge from the server. + # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) + # ":" unq(nonce-value) ":" unq(cnonce-value) + if self.algorithm == 'MD5-sess': + ha1 = H('%s:%s:%s' % (ha1, self.nonce, self.cnonce)) + + digest = H('%s:%s' % (ha1, req)) + return digest + + + +def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, stale=False): + """Constructs a WWW-Authenticate header for Digest authentication.""" + if qop not in valid_qops: + raise ValueError("Unsupported value for qop: '%s'" % qop) + if algorithm not in valid_algorithms: + raise ValueError("Unsupported value for algorithm: '%s'" % algorithm) + + if nonce is None: + nonce = synthesize_nonce(realm, key) + s = 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( + realm, nonce, algorithm, qop) + if stale: + s += ', stale="true"' + return s + + +def digest_auth(realm, get_ha1, key, debug=False): + """A CherryPy tool which hooks at before_handler to perform + HTTP Digest Access Authentication, as specified in :rfc:`2617`. + + If the request has an 'authorization' header with a 'Digest' scheme, this + tool authenticates the credentials supplied in that header. If + the request has no 'authorization' header, or if it does but the scheme is + not "Digest", or if authentication fails, the tool sends a 401 response with + a 'WWW-Authenticate' Digest header. + + realm + A string containing the authentication realm. + + get_ha1 + A callable which looks up a username in a credentials store + and returns the HA1 string, which is defined in the RFC to be + MD5(username : realm : password). The function's signature is: + ``get_ha1(realm, username)`` + where username is obtained from the request's 'authorization' header. + If username is not found in the credentials store, get_ha1() returns + None. + + key + A secret string known only to the server, used in the synthesis of nonces. + + """ + request = cherrypy.serving.request + + auth_header = request.headers.get('authorization') + nonce_is_stale = False + if auth_header is not None: + try: + auth = HttpDigestAuthorization(auth_header, request.method, debug=debug) + except ValueError: + raise cherrypy.HTTPError(400, "The Authorization header could not be parsed.") + + if debug: + TRACE(str(auth)) + + if auth.validate_nonce(realm, key): + ha1 = get_ha1(realm, auth.username) + if ha1 is not None: + # note that for request.body to be available we need to hook in at + # before_handler, not on_start_resource like 3.1.x digest_auth does. + digest = auth.request_digest(ha1, entity_body=request.body) + if digest == auth.response: # authenticated + if debug: + TRACE("digest matches auth.response") + # Now check if nonce is stale. + # The choice of ten minutes' lifetime for nonce is somewhat arbitrary + nonce_is_stale = auth.is_nonce_stale(max_age_seconds=600) + if not nonce_is_stale: + request.login = auth.username + if debug: + TRACE("authentication of %s successful" % auth.username) + return + + # Respond with 401 status and a WWW-Authenticate header + header = www_authenticate(realm, key, stale=nonce_is_stale) + if debug: + TRACE(header) + cherrypy.serving.response.headers['WWW-Authenticate'] = header + raise cherrypy.HTTPError(401, "You are not authorized to access that resource") + diff --git a/cherrypy/lib/caching.py b/cherrypy/lib/caching.py new file mode 100755 index 0000000..435b9dc --- /dev/null +++ b/cherrypy/lib/caching.py @@ -0,0 +1,465 @@ +""" +CherryPy implements a simple caching system as a pluggable Tool. This tool tries +to be an (in-process) HTTP/1.1-compliant cache. It's not quite there yet, but +it's probably good enough for most sites. + +In general, GET responses are cached (along with selecting headers) and, if +another request arrives for the same resource, the caching Tool will return 304 +Not Modified if possible, or serve the cached response otherwise. It also sets +request.cached to True if serving a cached representation, and sets +request.cacheable to False (so it doesn't get cached again). + +If POST, PUT, or DELETE requests are made for a cached resource, they invalidate +(delete) any cached response. + +Usage +===== + +Configuration file example:: + + [/] + tools.caching.on = True + tools.caching.delay = 3600 + +You may use a class other than the default +:class:`MemoryCache<cherrypy.lib.caching.MemoryCache>` by supplying the config +entry ``cache_class``; supply the full dotted name of the replacement class +as the config value. It must implement the basic methods ``get``, ``put``, +``delete``, and ``clear``. + +You may set any attribute, including overriding methods, on the cache +instance by providing them in config. The above sets the +:attr:`delay<cherrypy.lib.caching.MemoryCache.delay>` attribute, for example. +""" + +import datetime +import sys +import threading +import time + +import cherrypy +from cherrypy.lib import cptools, httputil +from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted + + +class Cache(object): + """Base class for Cache implementations.""" + + def get(self): + """Return the current variant if in the cache, else None.""" + raise NotImplemented + + def put(self, obj, size): + """Store the current variant in the cache.""" + raise NotImplemented + + def delete(self): + """Remove ALL cached variants of the current resource.""" + raise NotImplemented + + def clear(self): + """Reset the cache to its initial, empty state.""" + raise NotImplemented + + + +# ------------------------------- Memory Cache ------------------------------- # + + +class AntiStampedeCache(dict): + """A storage system for cached items which reduces stampede collisions.""" + + def wait(self, key, timeout=5, debug=False): + """Return the cached value for the given key, or None. + + If timeout is not None, and the value is already + being calculated by another thread, wait until the given timeout has + elapsed. If the value is available before the timeout expires, it is + returned. If not, None is returned, and a sentinel placed in the cache + to signal other threads to wait. + + If timeout is None, no waiting is performed nor sentinels used. + """ + value = self.get(key) + if isinstance(value, threading._Event): + if timeout is None: + # Ignore the other thread and recalc it ourselves. + if debug: + cherrypy.log('No timeout', 'TOOLS.CACHING') + return None + + # Wait until it's done or times out. + if debug: + cherrypy.log('Waiting up to %s seconds' % timeout, 'TOOLS.CACHING') + value.wait(timeout) + if value.result is not None: + # The other thread finished its calculation. Use it. + if debug: + cherrypy.log('Result!', 'TOOLS.CACHING') + return value.result + # Timed out. Stick an Event in the slot so other threads wait + # on this one to finish calculating the value. + if debug: + cherrypy.log('Timed out', 'TOOLS.CACHING') + e = threading.Event() + e.result = None + dict.__setitem__(self, key, e) + + return None + elif value is None: + # Stick an Event in the slot so other threads wait + # on this one to finish calculating the value. + if debug: + cherrypy.log('Timed out', 'TOOLS.CACHING') + e = threading.Event() + e.result = None + dict.__setitem__(self, key, e) + return value + + def __setitem__(self, key, value): + """Set the cached value for the given key.""" + existing = self.get(key) + dict.__setitem__(self, key, value) + if isinstance(existing, threading._Event): + # Set Event.result so other threads waiting on it have + # immediate access without needing to poll the cache again. + existing.result = value + existing.set() + + +class MemoryCache(Cache): + """An in-memory cache for varying response content. + + Each key in self.store is a URI, and each value is an AntiStampedeCache. + The response for any given URI may vary based on the values of + "selecting request headers"; that is, those named in the Vary + response header. We assume the list of header names to be constant + for each URI throughout the lifetime of the application, and store + that list in ``self.store[uri].selecting_headers``. + + The items contained in ``self.store[uri]`` have keys which are tuples of + request header values (in the same order as the names in its + selecting_headers), and values which are the actual responses. + """ + + maxobjects = 1000 + """The maximum number of cached objects; defaults to 1000.""" + + maxobj_size = 100000 + """The maximum size of each cached object in bytes; defaults to 100 KB.""" + + maxsize = 10000000 + """The maximum size of the entire cache in bytes; defaults to 10 MB.""" + + delay = 600 + """Seconds until the cached content expires; defaults to 600 (10 minutes).""" + + antistampede_timeout = 5 + """Seconds to wait for other threads to release a cache lock.""" + + expire_freq = 0.1 + """Seconds to sleep between cache expiration sweeps.""" + + debug = False + + def __init__(self): + self.clear() + + # Run self.expire_cache in a separate daemon thread. + t = threading.Thread(target=self.expire_cache, name='expire_cache') + self.expiration_thread = t + set_daemon(t, True) + t.start() + + def clear(self): + """Reset the cache to its initial, empty state.""" + self.store = {} + self.expirations = {} + self.tot_puts = 0 + self.tot_gets = 0 + self.tot_hist = 0 + self.tot_expires = 0 + self.tot_non_modified = 0 + self.cursize = 0 + + def expire_cache(self): + """Continuously examine cached objects, expiring stale ones. + + This function is designed to be run in its own daemon thread, + referenced at ``self.expiration_thread``. + """ + # It's possible that "time" will be set to None + # arbitrarily, so we check "while time" to avoid exceptions. + # See tickets #99 and #180 for more information. + while time: + now = time.time() + # Must make a copy of expirations so it doesn't change size + # during iteration + for expiration_time, objects in copyitems(self.expirations): + if expiration_time <= now: + for obj_size, uri, sel_header_values in objects: + try: + del self.store[uri][tuple(sel_header_values)] + self.tot_expires += 1 + self.cursize -= obj_size + except KeyError: + # the key may have been deleted elsewhere + pass + del self.expirations[expiration_time] + time.sleep(self.expire_freq) + + def get(self): + """Return the current variant if in the cache, else None.""" + request = cherrypy.serving.request + self.tot_gets += 1 + + uri = cherrypy.url(qs=request.query_string) + uricache = self.store.get(uri) + if uricache is None: + return None + + header_values = [request.headers.get(h, '') + for h in uricache.selecting_headers] + variant = uricache.wait(key=tuple(sorted(header_values)), + timeout=self.antistampede_timeout, + debug=self.debug) + if variant is not None: + self.tot_hist += 1 + return variant + + def put(self, variant, size): + """Store the current variant in the cache.""" + request = cherrypy.serving.request + response = cherrypy.serving.response + + uri = cherrypy.url(qs=request.query_string) + uricache = self.store.get(uri) + if uricache is None: + uricache = AntiStampedeCache() + uricache.selecting_headers = [ + e.value for e in response.headers.elements('Vary')] + self.store[uri] = uricache + + if len(self.store) < self.maxobjects: + total_size = self.cursize + size + + # checks if there's space for the object + if (size < self.maxobj_size and total_size < self.maxsize): + # add to the expirations list + expiration_time = response.time + self.delay + bucket = self.expirations.setdefault(expiration_time, []) + bucket.append((size, uri, uricache.selecting_headers)) + + # add to the cache + header_values = [request.headers.get(h, '') + for h in uricache.selecting_headers] + uricache[tuple(sorted(header_values))] = variant + self.tot_puts += 1 + self.cursize = total_size + + def delete(self): + """Remove ALL cached variants of the current resource.""" + uri = cherrypy.url(qs=cherrypy.serving.request.query_string) + self.store.pop(uri, None) + + +def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): + """Try to obtain cached output. If fresh enough, raise HTTPError(304). + + If POST, PUT, or DELETE: + * invalidates (deletes) any cached response for this resource + * sets request.cached = False + * sets request.cacheable = False + + else if a cached copy exists: + * sets request.cached = True + * sets request.cacheable = False + * sets response.headers to the cached values + * checks the cached Last-Modified response header against the + current If-(Un)Modified-Since request headers; raises 304 + if necessary. + * sets response.status and response.body to the cached values + * returns True + + otherwise: + * sets request.cached = False + * sets request.cacheable = True + * returns False + """ + request = cherrypy.serving.request + response = cherrypy.serving.response + + if not hasattr(cherrypy, "_cache"): + # Make a process-wide Cache object. + cherrypy._cache = kwargs.pop("cache_class", MemoryCache)() + + # Take all remaining kwargs and set them on the Cache object. + for k, v in kwargs.items(): + setattr(cherrypy._cache, k, v) + cherrypy._cache.debug = debug + + # POST, PUT, DELETE should invalidate (delete) the cached copy. + # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10. + if request.method in invalid_methods: + if debug: + cherrypy.log('request.method %r in invalid_methods %r' % + (request.method, invalid_methods), 'TOOLS.CACHING') + cherrypy._cache.delete() + request.cached = False + request.cacheable = False + return False + + if 'no-cache' in [e.value for e in request.headers.elements('Pragma')]: + request.cached = False + request.cacheable = True + return False + + cache_data = cherrypy._cache.get() + request.cached = bool(cache_data) + request.cacheable = not request.cached + if request.cached: + # Serve the cached copy. + max_age = cherrypy._cache.delay + for v in [e.value for e in request.headers.elements('Cache-Control')]: + atoms = v.split('=', 1) + directive = atoms.pop(0) + if directive == 'max-age': + if len(atoms) != 1 or not atoms[0].isdigit(): + raise cherrypy.HTTPError(400, "Invalid Cache-Control header") + max_age = int(atoms[0]) + break + elif directive == 'no-cache': + if debug: + cherrypy.log('Ignoring cache due to Cache-Control: no-cache', + 'TOOLS.CACHING') + request.cached = False + request.cacheable = True + return False + + if debug: + cherrypy.log('Reading response from cache', 'TOOLS.CACHING') + s, h, b, create_time = cache_data + age = int(response.time - create_time) + if (age > max_age): + if debug: + cherrypy.log('Ignoring cache due to age > %d' % max_age, + 'TOOLS.CACHING') + request.cached = False + request.cacheable = True + return False + + # Copy the response headers. See http://www.cherrypy.org/ticket/721. + response.headers = rh = httputil.HeaderMap() + for k in h: + dict.__setitem__(rh, k, dict.__getitem__(h, k)) + + # Add the required Age header + response.headers["Age"] = str(age) + + try: + # Note that validate_since depends on a Last-Modified header; + # this was put into the cached copy, and should have been + # resurrected just above (response.headers = cache_data[1]). + cptools.validate_since() + except cherrypy.HTTPRedirect: + x = sys.exc_info()[1] + if x.status == 304: + cherrypy._cache.tot_non_modified += 1 + raise + + # serve it & get out from the request + response.status = s + response.body = b + else: + if debug: + cherrypy.log('request is not cached', 'TOOLS.CACHING') + return request.cached + + +def tee_output(): + """Tee response output to cache storage. Internal.""" + # Used by CachingTool by attaching to request.hooks + + request = cherrypy.serving.request + if 'no-store' in request.headers.values('Cache-Control'): + return + + def tee(body): + """Tee response.body into a list.""" + if ('no-cache' in response.headers.values('Pragma') or + 'no-store' in response.headers.values('Cache-Control')): + for chunk in body: + yield chunk + return + + output = [] + for chunk in body: + output.append(chunk) + yield chunk + + # save the cache data + body = ntob('').join(output) + cherrypy._cache.put((response.status, response.headers or {}, + body, response.time), len(body)) + + response = cherrypy.serving.response + response.body = tee(response.body) + + +def expires(secs=0, force=False, debug=False): + """Tool for influencing cache mechanisms using the 'Expires' header. + + secs + Must be either an int or a datetime.timedelta, and indicates the + number of seconds between response.time and when the response should + expire. The 'Expires' header will be set to response.time + secs. + If secs is zero, the 'Expires' header is set one year in the past, and + the following "cache prevention" headers are also set: + + * Pragma: no-cache + * Cache-Control': no-cache, must-revalidate + + force + If False, the following headers are checked: + + * Etag + * Last-Modified + * Age + * Expires + + If any are already present, none of the above response headers are set. + + """ + + response = cherrypy.serving.response + headers = response.headers + + cacheable = False + if not force: + # some header names that indicate that the response can be cached + for indicator in ('Etag', 'Last-Modified', 'Age', 'Expires'): + if indicator in headers: + cacheable = True + break + + if not cacheable and not force: + if debug: + cherrypy.log('request is not cacheable', 'TOOLS.EXPIRES') + else: + if debug: + cherrypy.log('request is cacheable', 'TOOLS.EXPIRES') + if isinstance(secs, datetime.timedelta): + secs = (86400 * secs.days) + secs.seconds + + if secs == 0: + if force or ("Pragma" not in headers): + headers["Pragma"] = "no-cache" + if cherrypy.serving.request.protocol >= (1, 1): + if force or "Cache-Control" not in headers: + headers["Cache-Control"] = "no-cache, must-revalidate" + # Set an explicit Expires date in the past. + expiry = httputil.HTTPDate(1169942400.0) + else: + expiry = httputil.HTTPDate(response.time + secs) + if force or "Expires" not in headers: + headers["Expires"] = expiry diff --git a/cherrypy/lib/covercp.py b/cherrypy/lib/covercp.py new file mode 100755 index 0000000..9b701b5 --- /dev/null +++ b/cherrypy/lib/covercp.py @@ -0,0 +1,365 @@ +"""Code-coverage tools for CherryPy. + +To use this module, or the coverage tools in the test suite, +you need to download 'coverage.py', either Gareth Rees' `original +implementation <http://www.garethrees.org/2001/12/04/python-coverage/>`_ +or Ned Batchelder's `enhanced version: +<http://www.nedbatchelder.com/code/modules/coverage.html>`_ + +To turn on coverage tracing, use the following code:: + + cherrypy.engine.subscribe('start', covercp.start) + +DO NOT subscribe anything on the 'start_thread' channel, as previously +recommended. Calling start once in the main thread should be sufficient +to start coverage on all threads. Calling start again in each thread +effectively clears any coverage data gathered up to that point. + +Run your code, then use the ``covercp.serve()`` function to browse the +results in a web browser. If you run this module from the command line, +it will call ``serve()`` for you. +""" + +import re +import sys +import cgi +from cherrypy._cpcompat import quote_plus +import os, os.path +localFile = os.path.join(os.path.dirname(__file__), "coverage.cache") + +the_coverage = None +try: + from coverage import coverage + the_coverage = coverage(data_file=localFile) + def start(): + the_coverage.start() +except ImportError: + # Setting the_coverage to None will raise errors + # that need to be trapped downstream. + the_coverage = None + + import warnings + warnings.warn("No code coverage will be performed; coverage.py could not be imported.") + + def start(): + pass +start.priority = 20 + +TEMPLATE_MENU = """<html> +<head> + <title>CherryPy Coverage Menu</title> + <style> + body {font: 9pt Arial, serif;} + #tree { + font-size: 8pt; + font-family: Andale Mono, monospace; + white-space: pre; + } + #tree a:active, a:focus { + background-color: black; + padding: 1px; + color: white; + border: 0px solid #9999FF; + -moz-outline-style: none; + } + .fail { color: red;} + .pass { color: #888;} + #pct { text-align: right;} + h3 { + font-size: small; + font-weight: bold; + font-style: italic; + margin-top: 5px; + } + input { border: 1px solid #ccc; padding: 2px; } + .directory { + color: #933; + font-style: italic; + font-weight: bold; + font-size: 10pt; + } + .file { + color: #400; + } + a { text-decoration: none; } + #crumbs { + color: white; + font-size: 8pt; + font-family: Andale Mono, monospace; + width: 100%; + background-color: black; + } + #crumbs a { + color: #f88; + } + #options { + line-height: 2.3em; + border: 1px solid black; + background-color: #eee; + padding: 4px; + } + #exclude { + width: 100%; + margin-bottom: 3px; + border: 1px solid #999; + } + #submit { + background-color: black; + color: white; + border: 0; + margin-bottom: -9px; + } + </style> +</head> +<body> +<h2>CherryPy Coverage</h2>""" + +TEMPLATE_FORM = """ +<div id="options"> +<form action='menu' method=GET> + <input type='hidden' name='base' value='%(base)s' /> + Show percentages <input type='checkbox' %(showpct)s name='showpct' value='checked' /><br /> + Hide files over <input type='text' id='pct' name='pct' value='%(pct)s' size='3' />%%<br /> + Exclude files matching<br /> + <input type='text' id='exclude' name='exclude' value='%(exclude)s' size='20' /> + <br /> + + <input type='submit' value='Change view' id="submit"/> +</form> +</div>""" + +TEMPLATE_FRAMESET = """<html> +<head><title>CherryPy coverage data</title></head> +<frameset cols='250, 1*'> + <frame src='menu?base=%s' /> + <frame name='main' src='' /> +</frameset> +</html> +""" + +TEMPLATE_COVERAGE = """<html> +<head> + <title>Coverage for %(name)s</title> + <style> + h2 { margin-bottom: .25em; } + p { margin: .25em; } + .covered { color: #000; background-color: #fff; } + .notcovered { color: #fee; background-color: #500; } + .excluded { color: #00f; background-color: #fff; } + table .covered, table .notcovered, table .excluded + { font-family: Andale Mono, monospace; + font-size: 10pt; white-space: pre; } + + .lineno { background-color: #eee;} + .notcovered .lineno { background-color: #000;} + table { border-collapse: collapse; + </style> +</head> +<body> +<h2>%(name)s</h2> +<p>%(fullpath)s</p> +<p>Coverage: %(pc)s%%</p>""" + +TEMPLATE_LOC_COVERED = """<tr class="covered"> + <td class="lineno">%s </td> + <td>%s</td> +</tr>\n""" +TEMPLATE_LOC_NOT_COVERED = """<tr class="notcovered"> + <td class="lineno">%s </td> + <td>%s</td> +</tr>\n""" +TEMPLATE_LOC_EXCLUDED = """<tr class="excluded"> + <td class="lineno">%s </td> + <td>%s</td> +</tr>\n""" + +TEMPLATE_ITEM = "%s%s<a class='file' href='report?name=%s' target='main'>%s</a>\n" + +def _percent(statements, missing): + s = len(statements) + e = s - len(missing) + if s > 0: + return int(round(100.0 * e / s)) + return 0 + +def _show_branch(root, base, path, pct=0, showpct=False, exclude="", + coverage=the_coverage): + + # Show the directory name and any of our children + dirs = [k for k, v in root.items() if v] + dirs.sort() + for name in dirs: + newpath = os.path.join(path, name) + + if newpath.lower().startswith(base): + relpath = newpath[len(base):] + yield "| " * relpath.count(os.sep) + yield "<a class='directory' href='menu?base=%s&exclude=%s'>%s</a>\n" % \ + (newpath, quote_plus(exclude), name) + + for chunk in _show_branch(root[name], base, newpath, pct, showpct, exclude, coverage=coverage): + yield chunk + + # Now list the files + if path.lower().startswith(base): + relpath = path[len(base):] + files = [k for k, v in root.items() if not v] + files.sort() + for name in files: + newpath = os.path.join(path, name) + + pc_str = "" + if showpct: + try: + _, statements, _, missing, _ = coverage.analysis2(newpath) + except: + # Yes, we really want to pass on all errors. + pass + else: + pc = _percent(statements, missing) + pc_str = ("%3d%% " % pc).replace(' ',' ') + if pc < float(pct) or pc == -1: + pc_str = "<span class='fail'>%s</span>" % pc_str + else: + pc_str = "<span class='pass'>%s</span>" % pc_str + + yield TEMPLATE_ITEM % ("| " * (relpath.count(os.sep) + 1), + pc_str, newpath, name) + +def _skip_file(path, exclude): + if exclude: + return bool(re.search(exclude, path)) + +def _graft(path, tree): + d = tree + + p = path + atoms = [] + while True: + p, tail = os.path.split(p) + if not tail: + break + atoms.append(tail) + atoms.append(p) + if p != "/": + atoms.append("/") + + atoms.reverse() + for node in atoms: + if node: + d = d.setdefault(node, {}) + +def get_tree(base, exclude, coverage=the_coverage): + """Return covered module names as a nested dict.""" + tree = {} + runs = coverage.data.executed_files() + for path in runs: + if not _skip_file(path, exclude) and not os.path.isdir(path): + _graft(path, tree) + return tree + +class CoverStats(object): + + def __init__(self, coverage, root=None): + self.coverage = coverage + if root is None: + # Guess initial depth. Files outside this path will not be + # reachable from the web interface. + import cherrypy + root = os.path.dirname(cherrypy.__file__) + self.root = root + + def index(self): + return TEMPLATE_FRAMESET % self.root.lower() + index.exposed = True + + def menu(self, base="/", pct="50", showpct="", + exclude=r'python\d\.\d|test|tut\d|tutorial'): + + # The coverage module uses all-lower-case names. + base = base.lower().rstrip(os.sep) + + yield TEMPLATE_MENU + yield TEMPLATE_FORM % locals() + + # Start by showing links for parent paths + yield "<div id='crumbs'>" + path = "" + atoms = base.split(os.sep) + atoms.pop() + for atom in atoms: + path += atom + os.sep + yield ("<a href='menu?base=%s&exclude=%s'>%s</a> %s" + % (path, quote_plus(exclude), atom, os.sep)) + yield "</div>" + + yield "<div id='tree'>" + + # Then display the tree + tree = get_tree(base, exclude, self.coverage) + if not tree: + yield "<p>No modules covered.</p>" + else: + for chunk in _show_branch(tree, base, "/", pct, + showpct=='checked', exclude, coverage=self.coverage): + yield chunk + + yield "</div>" + yield "</body></html>" + menu.exposed = True + + def annotated_file(self, filename, statements, excluded, missing): + source = open(filename, 'r') + buffer = [] + for lineno, line in enumerate(source.readlines()): + lineno += 1 + line = line.strip("\n\r") + empty_the_buffer = True + if lineno in excluded: + template = TEMPLATE_LOC_EXCLUDED + elif lineno in missing: + template = TEMPLATE_LOC_NOT_COVERED + elif lineno in statements: + template = TEMPLATE_LOC_COVERED + else: + empty_the_buffer = False + buffer.append((lineno, line)) + if empty_the_buffer: + for lno, pastline in buffer: + yield template % (lno, cgi.escape(pastline)) + buffer = [] + yield template % (lineno, cgi.escape(line)) + + def report(self, name): + filename, statements, excluded, missing, _ = self.coverage.analysis2(name) + pc = _percent(statements, missing) + yield TEMPLATE_COVERAGE % dict(name=os.path.basename(name), + fullpath=name, + pc=pc) + yield '<table>\n' + for line in self.annotated_file(filename, statements, excluded, + missing): + yield line + yield '</table>' + yield '</body>' + yield '</html>' + report.exposed = True + + +def serve(path=localFile, port=8080, root=None): + if coverage is None: + raise ImportError("The coverage module could not be imported.") + from coverage import coverage + cov = coverage(data_file = path) + cov.load() + + import cherrypy + cherrypy.config.update({'server.socket_port': int(port), + 'server.thread_pool': 10, + 'environment': "production", + }) + cherrypy.quickstart(CoverStats(cov, root)) + +if __name__ == "__main__": + serve(*tuple(sys.argv[1:])) + diff --git a/cherrypy/lib/cpstats.py b/cherrypy/lib/cpstats.py new file mode 100755 index 0000000..79d5c3a --- /dev/null +++ b/cherrypy/lib/cpstats.py @@ -0,0 +1,661 @@ +"""CPStats, a package for collecting and reporting on program statistics. + +Overview +======== + +Statistics about program operation are an invaluable monitoring and debugging +tool. Unfortunately, the gathering and reporting of these critical values is +usually ad-hoc. This package aims to add a centralized place for gathering +statistical performance data, a structure for recording that data which +provides for extrapolation of that data into more useful information, +and a method of serving that data to both human investigators and +monitoring software. Let's examine each of those in more detail. + +Data Gathering +-------------- + +Just as Python's `logging` module provides a common importable for gathering +and sending messages, performance statistics would benefit from a similar +common mechanism, and one that does *not* require each package which wishes +to collect stats to import a third-party module. Therefore, we choose to +re-use the `logging` module by adding a `statistics` object to it. + +That `logging.statistics` object is a nested dict. It is not a custom class, +because that would 1) require libraries and applications to import a third- +party module in order to participate, 2) inhibit innovation in extrapolation +approaches and in reporting tools, and 3) be slow. There are, however, some +specifications regarding the structure of the dict. + + { + +----"SQLAlchemy": { + | "Inserts": 4389745, + | "Inserts per Second": + | lambda s: s["Inserts"] / (time() - s["Start"]), + | C +---"Table Statistics": { + | o | "widgets": {-----------+ + N | l | "Rows": 1.3M, | Record + a | l | "Inserts": 400, | + m | e | },---------------------+ + e | c | "froobles": { + s | t | "Rows": 7845, + p | i | "Inserts": 0, + a | o | }, + c | n +---}, + e | "Slow Queries": + | [{"Query": "SELECT * FROM widgets;", + | "Processing Time": 47.840923343, + | }, + | ], + +----}, + } + +The `logging.statistics` dict has four levels. The topmost level is nothing +more than a set of names to introduce modularity, usually along the lines of +package names. If the SQLAlchemy project wanted to participate, for example, +it might populate the item `logging.statistics['SQLAlchemy']`, whose value +would be a second-layer dict we call a "namespace". Namespaces help multiple +packages to avoid collisions over key names, and make reports easier to read, +to boot. The maintainers of SQLAlchemy should feel free to use more than one +namespace if needed (such as 'SQLAlchemy ORM'). Note that there are no case +or other syntax constraints on the namespace names; they should be chosen +to be maximally readable by humans (neither too short nor too long). + +Each namespace, then, is a dict of named statistical values, such as +'Requests/sec' or 'Uptime'. You should choose names which will look +good on a report: spaces and capitalization are just fine. + +In addition to scalars, values in a namespace MAY be a (third-layer) +dict, or a list, called a "collection". For example, the CherryPy StatsTool +keeps track of what each request is doing (or has most recently done) +in a 'Requests' collection, where each key is a thread ID; each +value in the subdict MUST be a fourth dict (whew!) of statistical data about +each thread. We call each subdict in the collection a "record". Similarly, +the StatsTool also keeps a list of slow queries, where each record contains +data about each slow query, in order. + +Values in a namespace or record may also be functions, which brings us to: + +Extrapolation +------------- + +The collection of statistical data needs to be fast, as close to unnoticeable +as possible to the host program. That requires us to minimize I/O, for example, +but in Python it also means we need to minimize function calls. So when you +are designing your namespace and record values, try to insert the most basic +scalar values you already have on hand. + +When it comes time to report on the gathered data, however, we usually have +much more freedom in what we can calculate. Therefore, whenever reporting +tools (like the provided StatsPage CherryPy class) fetch the contents of +`logging.statistics` for reporting, they first call `extrapolate_statistics` +(passing the whole `statistics` dict as the only argument). This makes a +deep copy of the statistics dict so that the reporting tool can both iterate +over it and even change it without harming the original. But it also expands +any functions in the dict by calling them. For example, you might have a +'Current Time' entry in the namespace with the value "lambda scope: time.time()". +The "scope" parameter is the current namespace dict (or record, if we're +currently expanding one of those instead), allowing you access to existing +static entries. If you're truly evil, you can even modify more than one entry +at a time. + +However, don't try to calculate an entry and then use its value in further +extrapolations; the order in which the functions are called is not guaranteed. +This can lead to a certain amount of duplicated work (or a redesign of your +schema), but that's better than complicating the spec. + +After the whole thing has been extrapolated, it's time for: + +Reporting +--------- + +The StatsPage class grabs the `logging.statistics` dict, extrapolates it all, +and then transforms it to HTML for easy viewing. Each namespace gets its own +header and attribute table, plus an extra table for each collection. This is +NOT part of the statistics specification; other tools can format how they like. + +You can control which columns are output and how they are formatted by updating +StatsPage.formatting, which is a dict that mirrors the keys and nesting of +`logging.statistics`. The difference is that, instead of data values, it has +formatting values. Use None for a given key to indicate to the StatsPage that a +given column should not be output. Use a string with formatting (such as '%.3f') +to interpolate the value(s), or use a callable (such as lambda v: v.isoformat()) +for more advanced formatting. Any entry which is not mentioned in the formatting +dict is output unchanged. + +Monitoring +---------- + +Although the HTML output takes pains to assign unique id's to each <td> with +statistical data, you're probably better off fetching /cpstats/data, which +outputs the whole (extrapolated) `logging.statistics` dict in JSON format. +That is probably easier to parse, and doesn't have any formatting controls, +so you get the "original" data in a consistently-serialized format. +Note: there's no treatment yet for datetime objects. Try time.time() instead +for now if you can. Nagios will probably thank you. + +Turning Collection Off +---------------------- + +It is recommended each namespace have an "Enabled" item which, if False, +stops collection (but not reporting) of statistical data. Applications +SHOULD provide controls to pause and resume collection by setting these +entries to False or True, if present. + + +Usage +===== + +To collect statistics on CherryPy applications: + + from cherrypy.lib import cpstats + appconfig['/']['tools.cpstats.on'] = True + +To collect statistics on your own code: + + import logging + # Initialize the repository + if not hasattr(logging, 'statistics'): logging.statistics = {} + # Initialize my namespace + mystats = logging.statistics.setdefault('My Stuff', {}) + # Initialize my namespace's scalars and collections + mystats.update({ + 'Enabled': True, + 'Start Time': time.time(), + 'Important Events': 0, + 'Events/Second': lambda s: ( + (s['Important Events'] / (time.time() - s['Start Time']))), + }) + ... + for event in events: + ... + # Collect stats + if mystats.get('Enabled', False): + mystats['Important Events'] += 1 + +To report statistics: + + root.cpstats = cpstats.StatsPage() + +To format statistics reports: + + See 'Reporting', above. + +""" + +# -------------------------------- Statistics -------------------------------- # + +import logging +if not hasattr(logging, 'statistics'): logging.statistics = {} + +def extrapolate_statistics(scope): + """Return an extrapolated copy of the given scope.""" + c = {} + for k, v in list(scope.items()): + if isinstance(v, dict): + v = extrapolate_statistics(v) + elif isinstance(v, (list, tuple)): + v = [extrapolate_statistics(record) for record in v] + elif hasattr(v, '__call__'): + v = v(scope) + c[k] = v + return c + + +# --------------------- CherryPy Applications Statistics --------------------- # + +import threading +import time + +import cherrypy + +appstats = logging.statistics.setdefault('CherryPy Applications', {}) +appstats.update({ + 'Enabled': True, + 'Bytes Read/Request': lambda s: (s['Total Requests'] and + (s['Total Bytes Read'] / float(s['Total Requests'])) or 0.0), + 'Bytes Read/Second': lambda s: s['Total Bytes Read'] / s['Uptime'](s), + 'Bytes Written/Request': lambda s: (s['Total Requests'] and + (s['Total Bytes Written'] / float(s['Total Requests'])) or 0.0), + 'Bytes Written/Second': lambda s: s['Total Bytes Written'] / s['Uptime'](s), + 'Current Time': lambda s: time.time(), + 'Current Requests': 0, + 'Requests/Second': lambda s: float(s['Total Requests']) / s['Uptime'](s), + 'Server Version': cherrypy.__version__, + 'Start Time': time.time(), + 'Total Bytes Read': 0, + 'Total Bytes Written': 0, + 'Total Requests': 0, + 'Total Time': 0, + 'Uptime': lambda s: time.time() - s['Start Time'], + 'Requests': {}, + }) + +proc_time = lambda s: time.time() - s['Start Time'] + + +class ByteCountWrapper(object): + """Wraps a file-like object, counting the number of bytes read.""" + + def __init__(self, rfile): + self.rfile = rfile + self.bytes_read = 0 + + def read(self, size=-1): + data = self.rfile.read(size) + self.bytes_read += len(data) + return data + + def readline(self, size=-1): + data = self.rfile.readline(size) + self.bytes_read += len(data) + return data + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline() + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline() + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def next(self): + data = self.rfile.next() + self.bytes_read += len(data) + return data + + +average_uriset_time = lambda s: s['Count'] and (s['Sum'] / s['Count']) or 0 + + +class StatsTool(cherrypy.Tool): + """Record various information about the current request.""" + + def __init__(self): + cherrypy.Tool.__init__(self, 'on_end_request', self.record_stop) + + def _setup(self): + """Hook this tool into cherrypy.request. + + The standard CherryPy request object will automatically call this + method when the tool is "turned on" in config. + """ + if appstats.get('Enabled', False): + cherrypy.Tool._setup(self) + self.record_start() + + def record_start(self): + """Record the beginning of a request.""" + request = cherrypy.serving.request + if not hasattr(request.rfile, 'bytes_read'): + request.rfile = ByteCountWrapper(request.rfile) + request.body.fp = request.rfile + + r = request.remote + + appstats['Current Requests'] += 1 + appstats['Total Requests'] += 1 + appstats['Requests'][threading._get_ident()] = { + 'Bytes Read': None, + 'Bytes Written': None, + # Use a lambda so the ip gets updated by tools.proxy later + 'Client': lambda s: '%s:%s' % (r.ip, r.port), + 'End Time': None, + 'Processing Time': proc_time, + 'Request-Line': request.request_line, + 'Response Status': None, + 'Start Time': time.time(), + } + + def record_stop(self, uriset=None, slow_queries=1.0, slow_queries_count=100, + debug=False, **kwargs): + """Record the end of a request.""" + w = appstats['Requests'][threading._get_ident()] + + r = cherrypy.request.rfile.bytes_read + w['Bytes Read'] = r + appstats['Total Bytes Read'] += r + + if cherrypy.response.stream: + w['Bytes Written'] = 'chunked' + else: + cl = int(cherrypy.response.headers.get('Content-Length', 0)) + w['Bytes Written'] = cl + appstats['Total Bytes Written'] += cl + + w['Response Status'] = cherrypy.response.status + + w['End Time'] = time.time() + p = w['End Time'] - w['Start Time'] + w['Processing Time'] = p + appstats['Total Time'] += p + + appstats['Current Requests'] -= 1 + + if debug: + cherrypy.log('Stats recorded: %s' % repr(w), 'TOOLS.CPSTATS') + + if uriset: + rs = appstats.setdefault('URI Set Tracking', {}) + r = rs.setdefault(uriset, { + 'Min': None, 'Max': None, 'Count': 0, 'Sum': 0, + 'Avg': average_uriset_time}) + if r['Min'] is None or p < r['Min']: + r['Min'] = p + if r['Max'] is None or p > r['Max']: + r['Max'] = p + r['Count'] += 1 + r['Sum'] += p + + if slow_queries and p > slow_queries: + sq = appstats.setdefault('Slow Queries', []) + sq.append(w.copy()) + if len(sq) > slow_queries_count: + sq.pop(0) + + +import cherrypy +cherrypy.tools.cpstats = StatsTool() + + +# ---------------------- CherryPy Statistics Reporting ---------------------- # + +import os +thisdir = os.path.abspath(os.path.dirname(__file__)) + +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + json = None + + +missing = object() + +locale_date = lambda v: time.strftime('%c', time.gmtime(v)) +iso_format = lambda v: time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) + +def pause_resume(ns): + def _pause_resume(enabled): + pause_disabled = '' + resume_disabled = '' + if enabled: + resume_disabled = 'disabled="disabled" ' + else: + pause_disabled = 'disabled="disabled" ' + return """ + <form action="pause" method="POST" style="display:inline"> + <input type="hidden" name="namespace" value="%s" /> + <input type="submit" value="Pause" %s/> + </form> + <form action="resume" method="POST" style="display:inline"> + <input type="hidden" name="namespace" value="%s" /> + <input type="submit" value="Resume" %s/> + </form> + """ % (ns, pause_disabled, ns, resume_disabled) + return _pause_resume + + +class StatsPage(object): + + formatting = { + 'CherryPy Applications': { + 'Enabled': pause_resume('CherryPy Applications'), + 'Bytes Read/Request': '%.3f', + 'Bytes Read/Second': '%.3f', + 'Bytes Written/Request': '%.3f', + 'Bytes Written/Second': '%.3f', + 'Current Time': iso_format, + 'Requests/Second': '%.3f', + 'Start Time': iso_format, + 'Total Time': '%.3f', + 'Uptime': '%.3f', + 'Slow Queries': { + 'End Time': None, + 'Processing Time': '%.3f', + 'Start Time': iso_format, + }, + 'URI Set Tracking': { + 'Avg': '%.3f', + 'Max': '%.3f', + 'Min': '%.3f', + 'Sum': '%.3f', + }, + 'Requests': { + 'Bytes Read': '%s', + 'Bytes Written': '%s', + 'End Time': None, + 'Processing Time': '%.3f', + 'Start Time': None, + }, + }, + 'CherryPy WSGIServer': { + 'Enabled': pause_resume('CherryPy WSGIServer'), + 'Connections/second': '%.3f', + 'Start time': iso_format, + }, + } + + + def index(self): + # Transform the raw data into pretty output for HTML + yield """ +<html> +<head> + <title>Statistics</title> +<style> + +th, td { + padding: 0.25em 0.5em; + border: 1px solid #666699; +} + +table { + border-collapse: collapse; +} + +table.stats1 { + width: 100%; +} + +table.stats1 th { + font-weight: bold; + text-align: right; + background-color: #CCD5DD; +} + +table.stats2, h2 { + margin-left: 50px; +} + +table.stats2 th { + font-weight: bold; + text-align: center; + background-color: #CCD5DD; +} + +</style> +</head> +<body> +""" + for title, scalars, collections in self.get_namespaces(): + yield """ +<h1>%s</h1> + +<table class='stats1'> + <tbody> +""" % title + for i, (key, value) in enumerate(scalars): + colnum = i % 3 + if colnum == 0: yield """ + <tr>""" + yield """ + <th>%(key)s</th><td id='%(title)s-%(key)s'>%(value)s</td>""" % vars() + if colnum == 2: yield """ + </tr>""" + + if colnum == 0: yield """ + <th></th><td></td> + <th></th><td></td> + </tr>""" + elif colnum == 1: yield """ + <th></th><td></td> + </tr>""" + yield """ + </tbody> +</table>""" + + for subtitle, headers, subrows in collections: + yield """ +<h2>%s</h2> +<table class='stats2'> + <thead> + <tr>""" % subtitle + for key in headers: + yield """ + <th>%s</th>""" % key + yield """ + </tr> + </thead> + <tbody>""" + for subrow in subrows: + yield """ + <tr>""" + for value in subrow: + yield """ + <td>%s</td>""" % value + yield """ + </tr>""" + yield """ + </tbody> +</table>""" + yield """ +</body> +</html> +""" + index.exposed = True + + def get_namespaces(self): + """Yield (title, scalars, collections) for each namespace.""" + s = extrapolate_statistics(logging.statistics) + for title, ns in sorted(s.items()): + scalars = [] + collections = [] + ns_fmt = self.formatting.get(title, {}) + for k, v in sorted(ns.items()): + fmt = ns_fmt.get(k, {}) + if isinstance(v, dict): + headers, subrows = self.get_dict_collection(v, fmt) + collections.append((k, ['ID'] + headers, subrows)) + elif isinstance(v, (list, tuple)): + headers, subrows = self.get_list_collection(v, fmt) + collections.append((k, headers, subrows)) + else: + format = ns_fmt.get(k, missing) + if format is None: + # Don't output this column. + continue + if hasattr(format, '__call__'): + v = format(v) + elif format is not missing: + v = format % v + scalars.append((k, v)) + yield title, scalars, collections + + def get_dict_collection(self, v, formatting): + """Return ([headers], [rows]) for the given collection.""" + # E.g., the 'Requests' dict. + headers = [] + for record in v.itervalues(): + for k3 in record: + format = formatting.get(k3, missing) + if format is None: + # Don't output this column. + continue + if k3 not in headers: + headers.append(k3) + headers.sort() + + subrows = [] + for k2, record in sorted(v.items()): + subrow = [k2] + for k3 in headers: + v3 = record.get(k3, '') + format = formatting.get(k3, missing) + if format is None: + # Don't output this column. + continue + if hasattr(format, '__call__'): + v3 = format(v3) + elif format is not missing: + v3 = format % v3 + subrow.append(v3) + subrows.append(subrow) + + return headers, subrows + + def get_list_collection(self, v, formatting): + """Return ([headers], [subrows]) for the given collection.""" + # E.g., the 'Slow Queries' list. + headers = [] + for record in v: + for k3 in record: + format = formatting.get(k3, missing) + if format is None: + # Don't output this column. + continue + if k3 not in headers: + headers.append(k3) + headers.sort() + + subrows = [] + for record in v: + subrow = [] + for k3 in headers: + v3 = record.get(k3, '') + format = formatting.get(k3, missing) + if format is None: + # Don't output this column. + continue + if hasattr(format, '__call__'): + v3 = format(v3) + elif format is not missing: + v3 = format % v3 + subrow.append(v3) + subrows.append(subrow) + + return headers, subrows + + if json is not None: + def data(self): + s = extrapolate_statistics(logging.statistics) + cherrypy.response.headers['Content-Type'] = 'application/json' + return json.dumps(s, sort_keys=True, indent=4) + data.exposed = True + + def pause(self, namespace): + logging.statistics.get(namespace, {})['Enabled'] = False + raise cherrypy.HTTPRedirect('./') + pause.exposed = True + pause.cp_config = {'tools.allow.on': True, + 'tools.allow.methods': ['POST']} + + def resume(self, namespace): + logging.statistics.get(namespace, {})['Enabled'] = True + raise cherrypy.HTTPRedirect('./') + resume.exposed = True + resume.cp_config = {'tools.allow.on': True, + 'tools.allow.methods': ['POST']} + diff --git a/cherrypy/lib/cptools.py b/cherrypy/lib/cptools.py new file mode 100755 index 0000000..3eedf97 --- /dev/null +++ b/cherrypy/lib/cptools.py @@ -0,0 +1,611 @@ +"""Functions for builtin CherryPy tools.""" + +import logging +import re + +import cherrypy +from cherrypy._cpcompat import basestring, ntob, md5, set +from cherrypy.lib import httputil as _httputil + + +# Conditional HTTP request support # + +def validate_etags(autotags=False, debug=False): + """Validate the current ETag against If-Match, If-None-Match headers. + + If autotags is True, an ETag response-header value will be provided + from an MD5 hash of the response body (unless some other code has + already provided an ETag header). If False (the default), the ETag + will not be automatic. + + WARNING: the autotags feature is not designed for URL's which allow + methods other than GET. For example, if a POST to the same URL returns + no content, the automatic ETag will be incorrect, breaking a fundamental + use for entity tags in a possibly destructive fashion. Likewise, if you + raise 304 Not Modified, the response body will be empty, the ETag hash + will be incorrect, and your application will break. + See :rfc:`2616` Section 14.24. + """ + response = cherrypy.serving.response + + # Guard against being run twice. + if hasattr(response, "ETag"): + return + + status, reason, msg = _httputil.valid_status(response.status) + + etag = response.headers.get('ETag') + + # Automatic ETag generation. See warning in docstring. + if etag: + if debug: + cherrypy.log('ETag already set: %s' % etag, 'TOOLS.ETAGS') + elif not autotags: + if debug: + cherrypy.log('Autotags off', 'TOOLS.ETAGS') + elif status != 200: + if debug: + cherrypy.log('Status not 200', 'TOOLS.ETAGS') + else: + etag = response.collapse_body() + etag = '"%s"' % md5(etag).hexdigest() + if debug: + cherrypy.log('Setting ETag: %s' % etag, 'TOOLS.ETAGS') + response.headers['ETag'] = etag + + response.ETag = etag + + # "If the request would, without the If-Match header field, result in + # anything other than a 2xx or 412 status, then the If-Match header + # MUST be ignored." + if debug: + cherrypy.log('Status: %s' % status, 'TOOLS.ETAGS') + if status >= 200 and status <= 299: + request = cherrypy.serving.request + + conditions = request.headers.elements('If-Match') or [] + conditions = [str(x) for x in conditions] + if debug: + cherrypy.log('If-Match conditions: %s' % repr(conditions), + 'TOOLS.ETAGS') + if conditions and not (conditions == ["*"] or etag in conditions): + raise cherrypy.HTTPError(412, "If-Match failed: ETag %r did " + "not match %r" % (etag, conditions)) + + conditions = request.headers.elements('If-None-Match') or [] + conditions = [str(x) for x in conditions] + if debug: + cherrypy.log('If-None-Match conditions: %s' % repr(conditions), + 'TOOLS.ETAGS') + if conditions == ["*"] or etag in conditions: + if debug: + cherrypy.log('request.method: %s' % request.method, 'TOOLS.ETAGS') + if request.method in ("GET", "HEAD"): + raise cherrypy.HTTPRedirect([], 304) + else: + raise cherrypy.HTTPError(412, "If-None-Match failed: ETag %r " + "matched %r" % (etag, conditions)) + +def validate_since(): + """Validate the current Last-Modified against If-Modified-Since headers. + + If no code has set the Last-Modified response header, then no validation + will be performed. + """ + response = cherrypy.serving.response + lastmod = response.headers.get('Last-Modified') + if lastmod: + status, reason, msg = _httputil.valid_status(response.status) + + request = cherrypy.serving.request + + since = request.headers.get('If-Unmodified-Since') + if since and since != lastmod: + if (status >= 200 and status <= 299) or status == 412: + raise cherrypy.HTTPError(412) + + since = request.headers.get('If-Modified-Since') + if since and since == lastmod: + if (status >= 200 and status <= 299) or status == 304: + if request.method in ("GET", "HEAD"): + raise cherrypy.HTTPRedirect([], 304) + else: + raise cherrypy.HTTPError(412) + + +# Tool code # + +def allow(methods=None, debug=False): + """Raise 405 if request.method not in methods (default GET/HEAD). + + The given methods are case-insensitive, and may be in any order. + If only one method is allowed, you may supply a single string; + if more than one, supply a list of strings. + + Regardless of whether the current method is allowed or not, this + also emits an 'Allow' response header, containing the given methods. + """ + if not isinstance(methods, (tuple, list)): + methods = [methods] + methods = [m.upper() for m in methods if m] + if not methods: + methods = ['GET', 'HEAD'] + elif 'GET' in methods and 'HEAD' not in methods: + methods.append('HEAD') + + cherrypy.response.headers['Allow'] = ', '.join(methods) + if cherrypy.request.method not in methods: + if debug: + cherrypy.log('request.method %r not in methods %r' % + (cherrypy.request.method, methods), 'TOOLS.ALLOW') + raise cherrypy.HTTPError(405) + else: + if debug: + cherrypy.log('request.method %r in methods %r' % + (cherrypy.request.method, methods), 'TOOLS.ALLOW') + + +def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', + scheme='X-Forwarded-Proto', debug=False): + """Change the base URL (scheme://host[:port][/path]). + + For running a CP server behind Apache, lighttpd, or other HTTP server. + + If you want the new request.base to include path info (not just the host), + you must explicitly set base to the full base path, and ALSO set 'local' + to '', so that the X-Forwarded-Host request header (which never includes + path info) does not override it. Regardless, the value for 'base' MUST + NOT end in a slash. + + cherrypy.request.remote.ip (the IP address of the client) will be + rewritten if the header specified by the 'remote' arg is valid. + By default, 'remote' is set to 'X-Forwarded-For'. If you do not + want to rewrite remote.ip, set the 'remote' arg to an empty string. + """ + + request = cherrypy.serving.request + + if scheme: + s = request.headers.get(scheme, None) + if debug: + cherrypy.log('Testing scheme %r:%r' % (scheme, s), 'TOOLS.PROXY') + if s == 'on' and 'ssl' in scheme.lower(): + # This handles e.g. webfaction's 'X-Forwarded-Ssl: on' header + scheme = 'https' + else: + # This is for lighttpd/pound/Mongrel's 'X-Forwarded-Proto: https' + scheme = s + if not scheme: + scheme = request.base[:request.base.find("://")] + + if local: + lbase = request.headers.get(local, None) + if debug: + cherrypy.log('Testing local %r:%r' % (local, lbase), 'TOOLS.PROXY') + if lbase is not None: + base = lbase.split(',')[0] + if not base: + port = request.local.port + if port == 80: + base = '127.0.0.1' + else: + base = '127.0.0.1:%s' % port + + if base.find("://") == -1: + # add http:// or https:// if needed + base = scheme + "://" + base + + request.base = base + + if remote: + xff = request.headers.get(remote) + if debug: + cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') + if xff: + if remote == 'X-Forwarded-For': + # See http://bob.pythonmac.org/archives/2005/09/23/apache-x-forwarded-for-caveat/ + xff = xff.split(',')[-1].strip() + request.remote.ip = xff + + +def ignore_headers(headers=('Range',), debug=False): + """Delete request headers whose field names are included in 'headers'. + + This is a useful tool for working behind certain HTTP servers; + for example, Apache duplicates the work that CP does for 'Range' + headers, and will doubly-truncate the response. + """ + request = cherrypy.serving.request + for name in headers: + if name in request.headers: + if debug: + cherrypy.log('Ignoring request header %r' % name, + 'TOOLS.IGNORE_HEADERS') + del request.headers[name] + + +def response_headers(headers=None, debug=False): + """Set headers on the response.""" + if debug: + cherrypy.log('Setting response headers: %s' % repr(headers), + 'TOOLS.RESPONSE_HEADERS') + for name, value in (headers or []): + cherrypy.serving.response.headers[name] = value +response_headers.failsafe = True + + +def referer(pattern, accept=True, accept_missing=False, error=403, + message='Forbidden Referer header.', debug=False): + """Raise HTTPError if Referer header does/does not match the given pattern. + + pattern + A regular expression pattern to test against the Referer. + + accept + If True, the Referer must match the pattern; if False, + the Referer must NOT match the pattern. + + accept_missing + If True, permit requests with no Referer header. + + error + The HTTP error code to return to the client on failure. + + message + A string to include in the response body on failure. + + """ + try: + ref = cherrypy.serving.request.headers['Referer'] + match = bool(re.match(pattern, ref)) + if debug: + cherrypy.log('Referer %r matches %r' % (ref, pattern), + 'TOOLS.REFERER') + if accept == match: + return + except KeyError: + if debug: + cherrypy.log('No Referer header', 'TOOLS.REFERER') + if accept_missing: + return + + raise cherrypy.HTTPError(error, message) + + +class SessionAuth(object): + """Assert that the user is logged in.""" + + session_key = "username" + debug = False + + def check_username_and_password(self, username, password): + pass + + def anonymous(self): + """Provide a temporary user name for anonymous users.""" + pass + + def on_login(self, username): + pass + + def on_logout(self, username): + pass + + def on_check(self, username): + pass + + def login_screen(self, from_page='..', username='', error_msg='', **kwargs): + return ntob("""<html><body> +Message: %(error_msg)s +<form method="post" action="do_login"> + Login: <input type="text" name="username" value="%(username)s" size="10" /><br /> + Password: <input type="password" name="password" size="10" /><br /> + <input type="hidden" name="from_page" value="%(from_page)s" /><br /> + <input type="submit" /> +</form> +</body></html>""" % {'from_page': from_page, 'username': username, + 'error_msg': error_msg}, "utf-8") + + def do_login(self, username, password, from_page='..', **kwargs): + """Login. May raise redirect, or return True if request handled.""" + response = cherrypy.serving.response + error_msg = self.check_username_and_password(username, password) + if error_msg: + body = self.login_screen(from_page, username, error_msg) + response.body = body + if "Content-Length" in response.headers: + # Delete Content-Length header so finalize() recalcs it. + del response.headers["Content-Length"] + return True + else: + cherrypy.serving.request.login = username + cherrypy.session[self.session_key] = username + self.on_login(username) + raise cherrypy.HTTPRedirect(from_page or "/") + + def do_logout(self, from_page='..', **kwargs): + """Logout. May raise redirect, or return True if request handled.""" + sess = cherrypy.session + username = sess.get(self.session_key) + sess[self.session_key] = None + if username: + cherrypy.serving.request.login = None + self.on_logout(username) + raise cherrypy.HTTPRedirect(from_page) + + def do_check(self): + """Assert username. May raise redirect, or return True if request handled.""" + sess = cherrypy.session + request = cherrypy.serving.request + response = cherrypy.serving.response + + username = sess.get(self.session_key) + if not username: + sess[self.session_key] = username = self.anonymous() + if self.debug: + cherrypy.log('No session[username], trying anonymous', 'TOOLS.SESSAUTH') + if not username: + url = cherrypy.url(qs=request.query_string) + if self.debug: + cherrypy.log('No username, routing to login_screen with ' + 'from_page %r' % url, 'TOOLS.SESSAUTH') + response.body = self.login_screen(url) + if "Content-Length" in response.headers: + # Delete Content-Length header so finalize() recalcs it. + del response.headers["Content-Length"] + return True + if self.debug: + cherrypy.log('Setting request.login to %r' % username, 'TOOLS.SESSAUTH') + request.login = username + self.on_check(username) + + def run(self): + request = cherrypy.serving.request + response = cherrypy.serving.response + + path = request.path_info + if path.endswith('login_screen'): + if self.debug: + cherrypy.log('routing %r to login_screen' % path, 'TOOLS.SESSAUTH') + return self.login_screen(**request.params) + elif path.endswith('do_login'): + if request.method != 'POST': + response.headers['Allow'] = "POST" + if self.debug: + cherrypy.log('do_login requires POST', 'TOOLS.SESSAUTH') + raise cherrypy.HTTPError(405) + if self.debug: + cherrypy.log('routing %r to do_login' % path, 'TOOLS.SESSAUTH') + return self.do_login(**request.params) + elif path.endswith('do_logout'): + if request.method != 'POST': + response.headers['Allow'] = "POST" + raise cherrypy.HTTPError(405) + if self.debug: + cherrypy.log('routing %r to do_logout' % path, 'TOOLS.SESSAUTH') + return self.do_logout(**request.params) + else: + if self.debug: + cherrypy.log('No special path, running do_check', 'TOOLS.SESSAUTH') + return self.do_check() + + +def session_auth(**kwargs): + sa = SessionAuth() + for k, v in kwargs.items(): + setattr(sa, k, v) + return sa.run() +session_auth.__doc__ = """Session authentication hook. + +Any attribute of the SessionAuth class may be overridden via a keyword arg +to this function: + +""" + "\n".join(["%s: %s" % (k, type(getattr(SessionAuth, k)).__name__) + for k in dir(SessionAuth) if not k.startswith("__")]) + + +def log_traceback(severity=logging.ERROR, debug=False): + """Write the last error's traceback to the cherrypy error log.""" + cherrypy.log("", "HTTP", severity=severity, traceback=True) + +def log_request_headers(debug=False): + """Write request headers to the cherrypy error log.""" + h = [" %s: %s" % (k, v) for k, v in cherrypy.serving.request.header_list] + cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP") + +def log_hooks(debug=False): + """Write request.hooks to the cherrypy error log.""" + request = cherrypy.serving.request + + msg = [] + # Sort by the standard points if possible. + from cherrypy import _cprequest + points = _cprequest.hookpoints + for k in request.hooks.keys(): + if k not in points: + points.append(k) + + for k in points: + msg.append(" %s:" % k) + v = request.hooks.get(k, []) + v.sort() + for h in v: + msg.append(" %r" % h) + cherrypy.log('\nRequest Hooks for ' + cherrypy.url() + + ':\n' + '\n'.join(msg), "HTTP") + +def redirect(url='', internal=True, debug=False): + """Raise InternalRedirect or HTTPRedirect to the given url.""" + if debug: + cherrypy.log('Redirecting %sto: %s' % + ({True: 'internal ', False: ''}[internal], url), + 'TOOLS.REDIRECT') + if internal: + raise cherrypy.InternalRedirect(url) + else: + raise cherrypy.HTTPRedirect(url) + +def trailing_slash(missing=True, extra=False, status=None, debug=False): + """Redirect if path_info has (missing|extra) trailing slash.""" + request = cherrypy.serving.request + pi = request.path_info + + if debug: + cherrypy.log('is_index: %r, missing: %r, extra: %r, path_info: %r' % + (request.is_index, missing, extra, pi), + 'TOOLS.TRAILING_SLASH') + if request.is_index is True: + if missing: + if not pi.endswith('/'): + new_url = cherrypy.url(pi + '/', request.query_string) + raise cherrypy.HTTPRedirect(new_url, status=status or 301) + elif request.is_index is False: + if extra: + # If pi == '/', don't redirect to ''! + if pi.endswith('/') and pi != '/': + new_url = cherrypy.url(pi[:-1], request.query_string) + raise cherrypy.HTTPRedirect(new_url, status=status or 301) + +def flatten(debug=False): + """Wrap response.body in a generator that recursively iterates over body. + + This allows cherrypy.response.body to consist of 'nested generators'; + that is, a set of generators that yield generators. + """ + import types + def flattener(input): + numchunks = 0 + for x in input: + if not isinstance(x, types.GeneratorType): + numchunks += 1 + yield x + else: + for y in flattener(x): + numchunks += 1 + yield y + if debug: + cherrypy.log('Flattened %d chunks' % numchunks, 'TOOLS.FLATTEN') + response = cherrypy.serving.response + response.body = flattener(response.body) + + +def accept(media=None, debug=False): + """Return the client's preferred media-type (from the given Content-Types). + + If 'media' is None (the default), no test will be performed. + + If 'media' is provided, it should be the Content-Type value (as a string) + or values (as a list or tuple of strings) which the current resource + can emit. The client's acceptable media ranges (as declared in the + Accept request header) will be matched in order to these Content-Type + values; the first such string is returned. That is, the return value + will always be one of the strings provided in the 'media' arg (or None + if 'media' is None). + + If no match is found, then HTTPError 406 (Not Acceptable) is raised. + Note that most web browsers send */* as a (low-quality) acceptable + media range, which should match any Content-Type. In addition, "...if + no Accept header field is present, then it is assumed that the client + accepts all media types." + + Matching types are checked in order of client preference first, + and then in the order of the given 'media' values. + + Note that this function does not honor accept-params (other than "q"). + """ + if not media: + return + if isinstance(media, basestring): + media = [media] + request = cherrypy.serving.request + + # Parse the Accept request header, and try to match one + # of the requested media-ranges (in order of preference). + ranges = request.headers.elements('Accept') + if not ranges: + # Any media type is acceptable. + if debug: + cherrypy.log('No Accept header elements', 'TOOLS.ACCEPT') + return media[0] + else: + # Note that 'ranges' is sorted in order of preference + for element in ranges: + if element.qvalue > 0: + if element.value == "*/*": + # Matches any type or subtype + if debug: + cherrypy.log('Match due to */*', 'TOOLS.ACCEPT') + return media[0] + elif element.value.endswith("/*"): + # Matches any subtype + mtype = element.value[:-1] # Keep the slash + for m in media: + if m.startswith(mtype): + if debug: + cherrypy.log('Match due to %s' % element.value, + 'TOOLS.ACCEPT') + return m + else: + # Matches exact value + if element.value in media: + if debug: + cherrypy.log('Match due to %s' % element.value, + 'TOOLS.ACCEPT') + return element.value + + # No suitable media-range found. + ah = request.headers.get('Accept') + if ah is None: + msg = "Your client did not send an Accept header." + else: + msg = "Your client sent this Accept header: %s." % ah + msg += (" But this resource only emits these media types: %s." % + ", ".join(media)) + raise cherrypy.HTTPError(406, msg) + + +class MonitoredHeaderMap(_httputil.HeaderMap): + + def __init__(self): + self.accessed_headers = set() + + def __getitem__(self, key): + self.accessed_headers.add(key) + return _httputil.HeaderMap.__getitem__(self, key) + + def __contains__(self, key): + self.accessed_headers.add(key) + return _httputil.HeaderMap.__contains__(self, key) + + def get(self, key, default=None): + self.accessed_headers.add(key) + return _httputil.HeaderMap.get(self, key, default=default) + + def has_key(self, key): + self.accessed_headers.add(key) + return _httputil.HeaderMap.has_key(self, key) + + +def autovary(ignore=None, debug=False): + """Auto-populate the Vary response header based on request.header access.""" + request = cherrypy.serving.request + + req_h = request.headers + request.headers = MonitoredHeaderMap() + request.headers.update(req_h) + if ignore is None: + ignore = set(['Content-Disposition', 'Content-Length', 'Content-Type']) + + def set_response_header(): + resp_h = cherrypy.serving.response.headers + v = set([e.value for e in resp_h.elements('Vary')]) + if debug: + cherrypy.log('Accessed headers: %s' % request.headers.accessed_headers, + 'TOOLS.AUTOVARY') + v = v.union(request.headers.accessed_headers) + v = v.difference(ignore) + v = list(v) + v.sort() + resp_h['Vary'] = ', '.join(v) + request.hooks.attach('before_finalize', set_response_header, 95) + diff --git a/cherrypy/lib/encoding.py b/cherrypy/lib/encoding.py new file mode 100755 index 0000000..6459746 --- /dev/null +++ b/cherrypy/lib/encoding.py @@ -0,0 +1,388 @@ +import struct +import time + +import cherrypy +from cherrypy._cpcompat import basestring, BytesIO, ntob, set, unicodestr +from cherrypy.lib import file_generator +from cherrypy.lib import set_vary_header + + +def decode(encoding=None, default_encoding='utf-8'): + """Replace or extend the list of charsets used to decode a request entity. + + Either argument may be a single string or a list of strings. + + encoding + If not None, restricts the set of charsets attempted while decoding + a request entity to the given set (even if a different charset is given in + the Content-Type request header). + + default_encoding + Only in effect if the 'encoding' argument is not given. + If given, the set of charsets attempted while decoding a request entity is + *extended* with the given value(s). + + """ + body = cherrypy.request.body + if encoding is not None: + if not isinstance(encoding, list): + encoding = [encoding] + body.attempt_charsets = encoding + elif default_encoding: + if not isinstance(default_encoding, list): + default_encoding = [default_encoding] + body.attempt_charsets = body.attempt_charsets + default_encoding + + +class ResponseEncoder: + + default_encoding = 'utf-8' + failmsg = "Response body could not be encoded with %r." + encoding = None + errors = 'strict' + text_only = True + add_charset = True + debug = False + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + self.attempted_charsets = set() + request = cherrypy.serving.request + if request.handler is not None: + # Replace request.handler with self + if self.debug: + cherrypy.log('Replacing request.handler', 'TOOLS.ENCODE') + self.oldhandler = request.handler + request.handler = self + + def encode_stream(self, encoding): + """Encode a streaming response body. + + Use a generator wrapper, and just pray it works as the stream is + being written out. + """ + if encoding in self.attempted_charsets: + return False + self.attempted_charsets.add(encoding) + + def encoder(body): + for chunk in body: + if isinstance(chunk, unicodestr): + chunk = chunk.encode(encoding, self.errors) + yield chunk + self.body = encoder(self.body) + return True + + def encode_string(self, encoding): + """Encode a buffered response body.""" + if encoding in self.attempted_charsets: + return False + self.attempted_charsets.add(encoding) + + try: + body = [] + for chunk in self.body: + if isinstance(chunk, unicodestr): + chunk = chunk.encode(encoding, self.errors) + body.append(chunk) + self.body = body + except (LookupError, UnicodeError): + return False + else: + return True + + def find_acceptable_charset(self): + request = cherrypy.serving.request + response = cherrypy.serving.response + + if self.debug: + cherrypy.log('response.stream %r' % response.stream, 'TOOLS.ENCODE') + if response.stream: + encoder = self.encode_stream + else: + encoder = self.encode_string + if "Content-Length" in response.headers: + # Delete Content-Length header so finalize() recalcs it. + # Encoded strings may be of different lengths from their + # unicode equivalents, and even from each other. For example: + # >>> t = u"\u7007\u3040" + # >>> len(t) + # 2 + # >>> len(t.encode("UTF-8")) + # 6 + # >>> len(t.encode("utf7")) + # 8 + del response.headers["Content-Length"] + + # Parse the Accept-Charset request header, and try to provide one + # of the requested charsets (in order of user preference). + encs = request.headers.elements('Accept-Charset') + charsets = [enc.value.lower() for enc in encs] + if self.debug: + cherrypy.log('charsets %s' % repr(charsets), 'TOOLS.ENCODE') + + if self.encoding is not None: + # If specified, force this encoding to be used, or fail. + encoding = self.encoding.lower() + if self.debug: + cherrypy.log('Specified encoding %r' % encoding, 'TOOLS.ENCODE') + if (not charsets) or "*" in charsets or encoding in charsets: + if self.debug: + cherrypy.log('Attempting encoding %r' % encoding, 'TOOLS.ENCODE') + if encoder(encoding): + return encoding + else: + if not encs: + if self.debug: + cherrypy.log('Attempting default encoding %r' % + self.default_encoding, 'TOOLS.ENCODE') + # Any character-set is acceptable. + if encoder(self.default_encoding): + return self.default_encoding + else: + raise cherrypy.HTTPError(500, self.failmsg % self.default_encoding) + else: + for element in encs: + if element.qvalue > 0: + if element.value == "*": + # Matches any charset. Try our default. + if self.debug: + cherrypy.log('Attempting default encoding due ' + 'to %r' % element, 'TOOLS.ENCODE') + if encoder(self.default_encoding): + return self.default_encoding + else: + encoding = element.value + if self.debug: + cherrypy.log('Attempting encoding %s (qvalue >' + '0)' % element, 'TOOLS.ENCODE') + if encoder(encoding): + return encoding + + if "*" not in charsets: + # If no "*" is present in an Accept-Charset field, then all + # character sets not explicitly mentioned get a quality + # value of 0, except for ISO-8859-1, which gets a quality + # value of 1 if not explicitly mentioned. + iso = 'iso-8859-1' + if iso not in charsets: + if self.debug: + cherrypy.log('Attempting ISO-8859-1 encoding', + 'TOOLS.ENCODE') + if encoder(iso): + return iso + + # No suitable encoding found. + ac = request.headers.get('Accept-Charset') + if ac is None: + msg = "Your client did not send an Accept-Charset header." + else: + msg = "Your client sent this Accept-Charset header: %s." % ac + msg += " We tried these charsets: %s." % ", ".join(self.attempted_charsets) + raise cherrypy.HTTPError(406, msg) + + def __call__(self, *args, **kwargs): + response = cherrypy.serving.response + self.body = self.oldhandler(*args, **kwargs) + + if isinstance(self.body, basestring): + # strings get wrapped in a list because iterating over a single + # item list is much faster than iterating over every character + # in a long string. + if self.body: + self.body = [self.body] + else: + # [''] doesn't evaluate to False, so replace it with []. + self.body = [] + elif hasattr(self.body, 'read'): + self.body = file_generator(self.body) + elif self.body is None: + self.body = [] + + ct = response.headers.elements("Content-Type") + if self.debug: + cherrypy.log('Content-Type: %r' % [str(h) for h in ct], 'TOOLS.ENCODE') + if ct: + ct = ct[0] + if self.text_only: + if ct.value.lower().startswith("text/"): + if self.debug: + cherrypy.log('Content-Type %s starts with "text/"' % ct, + 'TOOLS.ENCODE') + do_find = True + else: + if self.debug: + cherrypy.log('Not finding because Content-Type %s does ' + 'not start with "text/"' % ct, + 'TOOLS.ENCODE') + do_find = False + else: + if self.debug: + cherrypy.log('Finding because not text_only', 'TOOLS.ENCODE') + do_find = True + + if do_find: + # Set "charset=..." param on response Content-Type header + ct.params['charset'] = self.find_acceptable_charset() + if self.add_charset: + if self.debug: + cherrypy.log('Setting Content-Type %s' % ct, + 'TOOLS.ENCODE') + response.headers["Content-Type"] = str(ct) + + return self.body + +# GZIP + +def compress(body, compress_level): + """Compress 'body' at the given compress_level.""" + import zlib + + # See http://www.gzip.org/zlib/rfc-gzip.html + yield ntob('\x1f\x8b') # ID1 and ID2: gzip marker + yield ntob('\x08') # CM: compression method + yield ntob('\x00') # FLG: none set + # MTIME: 4 bytes + yield struct.pack("<L", int(time.time()) & int('FFFFFFFF', 16)) + yield ntob('\x02') # XFL: max compression, slowest algo + yield ntob('\xff') # OS: unknown + + crc = zlib.crc32(ntob("")) + size = 0 + zobj = zlib.compressobj(compress_level, + zlib.DEFLATED, -zlib.MAX_WBITS, + zlib.DEF_MEM_LEVEL, 0) + for line in body: + size += len(line) + crc = zlib.crc32(line, crc) + yield zobj.compress(line) + yield zobj.flush() + + # CRC32: 4 bytes + yield struct.pack("<L", crc & int('FFFFFFFF', 16)) + # ISIZE: 4 bytes + yield struct.pack("<L", size & int('FFFFFFFF', 16)) + +def decompress(body): + import gzip + + zbuf = BytesIO() + zbuf.write(body) + zbuf.seek(0) + zfile = gzip.GzipFile(mode='rb', fileobj=zbuf) + data = zfile.read() + zfile.close() + return data + + +def gzip(compress_level=5, mime_types=['text/html', 'text/plain'], debug=False): + """Try to gzip the response body if Content-Type in mime_types. + + cherrypy.response.headers['Content-Type'] must be set to one of the + values in the mime_types arg before calling this function. + + The provided list of mime-types must be of one of the following form: + * type/subtype + * type/* + * type/*+subtype + + No compression is performed if any of the following hold: + * The client sends no Accept-Encoding request header + * No 'gzip' or 'x-gzip' is present in the Accept-Encoding header + * No 'gzip' or 'x-gzip' with a qvalue > 0 is present + * The 'identity' value is given with a qvalue > 0. + + """ + request = cherrypy.serving.request + response = cherrypy.serving.response + + set_vary_header(response, "Accept-Encoding") + + if not response.body: + # Response body is empty (might be a 304 for instance) + if debug: + cherrypy.log('No response body', context='TOOLS.GZIP') + return + + # If returning cached content (which should already have been gzipped), + # don't re-zip. + if getattr(request, "cached", False): + if debug: + cherrypy.log('Not gzipping cached response', context='TOOLS.GZIP') + return + + acceptable = request.headers.elements('Accept-Encoding') + if not acceptable: + # If no Accept-Encoding field is present in a request, + # the server MAY assume that the client will accept any + # content coding. In this case, if "identity" is one of + # the available content-codings, then the server SHOULD use + # the "identity" content-coding, unless it has additional + # information that a different content-coding is meaningful + # to the client. + if debug: + cherrypy.log('No Accept-Encoding', context='TOOLS.GZIP') + return + + ct = response.headers.get('Content-Type', '').split(';')[0] + for coding in acceptable: + if coding.value == 'identity' and coding.qvalue != 0: + if debug: + cherrypy.log('Non-zero identity qvalue: %s' % coding, + context='TOOLS.GZIP') + return + if coding.value in ('gzip', 'x-gzip'): + if coding.qvalue == 0: + if debug: + cherrypy.log('Zero gzip qvalue: %s' % coding, + context='TOOLS.GZIP') + return + + if ct not in mime_types: + # If the list of provided mime-types contains tokens + # such as 'text/*' or 'application/*+xml', + # we go through them and find the most appropriate one + # based on the given content-type. + # The pattern matching is only caring about the most + # common cases, as stated above, and doesn't support + # for extra parameters. + found = False + if '/' in ct: + ct_media_type, ct_sub_type = ct.split('/') + for mime_type in mime_types: + if '/' in mime_type: + media_type, sub_type = mime_type.split('/') + if ct_media_type == media_type: + if sub_type == '*': + found = True + break + elif '+' in sub_type and '+' in ct_sub_type: + ct_left, ct_right = ct_sub_type.split('+') + left, right = sub_type.split('+') + if left == '*' and ct_right == right: + found = True + break + + if not found: + if debug: + cherrypy.log('Content-Type %s not in mime_types %r' % + (ct, mime_types), context='TOOLS.GZIP') + return + + if debug: + cherrypy.log('Gzipping', context='TOOLS.GZIP') + # Return a generator that compresses the page + response.headers['Content-Encoding'] = 'gzip' + response.body = compress(response.body, compress_level) + if "Content-Length" in response.headers: + # Delete Content-Length header so finalize() recalcs it. + del response.headers["Content-Length"] + + return + + if debug: + cherrypy.log('No acceptable encoding found.', context='GZIP') + cherrypy.HTTPError(406, "identity, gzip").set_response() + diff --git a/cherrypy/lib/http.py b/cherrypy/lib/http.py new file mode 100755 index 0000000..4661d69 --- /dev/null +++ b/cherrypy/lib/http.py @@ -0,0 +1,7 @@ +import warnings +warnings.warn('cherrypy.lib.http has been deprecated and will be removed ' + 'in CherryPy 3.3 use cherrypy.lib.httputil instead.', + DeprecationWarning) + +from cherrypy.lib.httputil import * + diff --git a/cherrypy/lib/httpauth.py b/cherrypy/lib/httpauth.py new file mode 100755 index 0000000..ad7c6eb --- /dev/null +++ b/cherrypy/lib/httpauth.py @@ -0,0 +1,354 @@ +""" +This module defines functions to implement HTTP Digest Authentication (:rfc:`2617`). +This has full compliance with 'Digest' and 'Basic' authentication methods. In +'Digest' it supports both MD5 and MD5-sess algorithms. + +Usage: + First use 'doAuth' to request the client authentication for a + certain resource. You should send an httplib.UNAUTHORIZED response to the + client so he knows he has to authenticate itself. + + Then use 'parseAuthorization' to retrieve the 'auth_map' used in + 'checkResponse'. + + To use 'checkResponse' you must have already verified the password associated + with the 'username' key in 'auth_map' dict. Then you use the 'checkResponse' + function to verify if the password matches the one sent by the client. + +SUPPORTED_ALGORITHM - list of supported 'Digest' algorithms +SUPPORTED_QOP - list of supported 'Digest' 'qop'. +""" +__version__ = 1, 0, 1 +__author__ = "Tiago Cogumbreiro <cogumbreiro@users.sf.net>" +__credits__ = """ + Peter van Kampen for its recipe which implement most of Digest authentication: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302378 +""" + +__license__ = """ +Copyright (c) 2005, Tiago Cogumbreiro <cogumbreiro@users.sf.net> +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Sylvain Hellegouarch nor the names of his contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +__all__ = ("digestAuth", "basicAuth", "doAuth", "checkResponse", + "parseAuthorization", "SUPPORTED_ALGORITHM", "md5SessionKey", + "calculateNonce", "SUPPORTED_QOP") + +################################################################################ +import time +from cherrypy._cpcompat import base64_decode, ntob, md5 +from cherrypy._cpcompat import parse_http_list, parse_keqv_list + +MD5 = "MD5" +MD5_SESS = "MD5-sess" +AUTH = "auth" +AUTH_INT = "auth-int" + +SUPPORTED_ALGORITHM = (MD5, MD5_SESS) +SUPPORTED_QOP = (AUTH, AUTH_INT) + +################################################################################ +# doAuth +# +DIGEST_AUTH_ENCODERS = { + MD5: lambda val: md5(ntob(val)).hexdigest(), + MD5_SESS: lambda val: md5(ntob(val)).hexdigest(), +# SHA: lambda val: sha.new(ntob(val)).hexdigest (), +} + +def calculateNonce (realm, algorithm = MD5): + """This is an auxaliary function that calculates 'nonce' value. It is used + to handle sessions.""" + + global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS + assert algorithm in SUPPORTED_ALGORITHM + + try: + encoder = DIGEST_AUTH_ENCODERS[algorithm] + except KeyError: + raise NotImplementedError ("The chosen algorithm (%s) does not have "\ + "an implementation yet" % algorithm) + + return encoder ("%d:%s" % (time.time(), realm)) + +def digestAuth (realm, algorithm = MD5, nonce = None, qop = AUTH): + """Challenges the client for a Digest authentication.""" + global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS, SUPPORTED_QOP + assert algorithm in SUPPORTED_ALGORITHM + assert qop in SUPPORTED_QOP + + if nonce is None: + nonce = calculateNonce (realm, algorithm) + + return 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( + realm, nonce, algorithm, qop + ) + +def basicAuth (realm): + """Challengenes the client for a Basic authentication.""" + assert '"' not in realm, "Realms cannot contain the \" (quote) character." + + return 'Basic realm="%s"' % realm + +def doAuth (realm): + """'doAuth' function returns the challenge string b giving priority over + Digest and fallback to Basic authentication when the browser doesn't + support the first one. + + This should be set in the HTTP header under the key 'WWW-Authenticate'.""" + + return digestAuth (realm) + " " + basicAuth (realm) + + +################################################################################ +# Parse authorization parameters +# +def _parseDigestAuthorization (auth_params): + # Convert the auth params to a dict + items = parse_http_list(auth_params) + params = parse_keqv_list(items) + + # Now validate the params + + # Check for required parameters + required = ["username", "realm", "nonce", "uri", "response"] + for k in required: + if k not in params: + return None + + # If qop is sent then cnonce and nc MUST be present + if "qop" in params and not ("cnonce" in params \ + and "nc" in params): + return None + + # If qop is not sent, neither cnonce nor nc can be present + if ("cnonce" in params or "nc" in params) and \ + "qop" not in params: + return None + + return params + + +def _parseBasicAuthorization (auth_params): + username, password = base64_decode(auth_params).split(":", 1) + return {"username": username, "password": password} + +AUTH_SCHEMES = { + "basic": _parseBasicAuthorization, + "digest": _parseDigestAuthorization, +} + +def parseAuthorization (credentials): + """parseAuthorization will convert the value of the 'Authorization' key in + the HTTP header to a map itself. If the parsing fails 'None' is returned. + """ + + global AUTH_SCHEMES + + auth_scheme, auth_params = credentials.split(" ", 1) + auth_scheme = auth_scheme.lower () + + parser = AUTH_SCHEMES[auth_scheme] + params = parser (auth_params) + + if params is None: + return + + assert "auth_scheme" not in params + params["auth_scheme"] = auth_scheme + return params + + +################################################################################ +# Check provided response for a valid password +# +def md5SessionKey (params, password): + """ + If the "algorithm" directive's value is "MD5-sess", then A1 + [the session key] is calculated only once - on the first request by the + client following receipt of a WWW-Authenticate challenge from the server. + + This creates a 'session key' for the authentication of subsequent + requests and responses which is different for each "authentication + session", thus limiting the amount of material hashed with any one + key. + + Because the server need only use the hash of the user + credentials in order to create the A1 value, this construction could + be used in conjunction with a third party authentication service so + that the web server would not need the actual password value. The + specification of such a protocol is beyond the scope of this + specification. +""" + + keys = ("username", "realm", "nonce", "cnonce") + params_copy = {} + for key in keys: + params_copy[key] = params[key] + + params_copy["algorithm"] = MD5_SESS + return _A1 (params_copy, password) + +def _A1(params, password): + algorithm = params.get ("algorithm", MD5) + H = DIGEST_AUTH_ENCODERS[algorithm] + + if algorithm == MD5: + # If the "algorithm" directive's value is "MD5" or is + # unspecified, then A1 is: + # A1 = unq(username-value) ":" unq(realm-value) ":" passwd + return "%s:%s:%s" % (params["username"], params["realm"], password) + + elif algorithm == MD5_SESS: + + # This is A1 if qop is set + # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) + # ":" unq(nonce-value) ":" unq(cnonce-value) + h_a1 = H ("%s:%s:%s" % (params["username"], params["realm"], password)) + return "%s:%s:%s" % (h_a1, params["nonce"], params["cnonce"]) + + +def _A2(params, method, kwargs): + # If the "qop" directive's value is "auth" or is unspecified, then A2 is: + # A2 = Method ":" digest-uri-value + + qop = params.get ("qop", "auth") + if qop == "auth": + return method + ":" + params["uri"] + elif qop == "auth-int": + # If the "qop" value is "auth-int", then A2 is: + # A2 = Method ":" digest-uri-value ":" H(entity-body) + entity_body = kwargs.get ("entity_body", "") + H = kwargs["H"] + + return "%s:%s:%s" % ( + method, + params["uri"], + H(entity_body) + ) + + else: + raise NotImplementedError ("The 'qop' method is unknown: %s" % qop) + +def _computeDigestResponse(auth_map, password, method = "GET", A1 = None,**kwargs): + """ + Generates a response respecting the algorithm defined in RFC 2617 + """ + params = auth_map + + algorithm = params.get ("algorithm", MD5) + + H = DIGEST_AUTH_ENCODERS[algorithm] + KD = lambda secret, data: H(secret + ":" + data) + + qop = params.get ("qop", None) + + H_A2 = H(_A2(params, method, kwargs)) + + if algorithm == MD5_SESS and A1 is not None: + H_A1 = H(A1) + else: + H_A1 = H(_A1(params, password)) + + if qop in ("auth", "auth-int"): + # If the "qop" value is "auth" or "auth-int": + # request-digest = <"> < KD ( H(A1), unq(nonce-value) + # ":" nc-value + # ":" unq(cnonce-value) + # ":" unq(qop-value) + # ":" H(A2) + # ) <"> + request = "%s:%s:%s:%s:%s" % ( + params["nonce"], + params["nc"], + params["cnonce"], + params["qop"], + H_A2, + ) + elif qop is None: + # If the "qop" directive is not present (this construction is + # for compatibility with RFC 2069): + # request-digest = + # <"> < KD ( H(A1), unq(nonce-value) ":" H(A2) ) > <"> + request = "%s:%s" % (params["nonce"], H_A2) + + return KD(H_A1, request) + +def _checkDigestResponse(auth_map, password, method = "GET", A1 = None, **kwargs): + """This function is used to verify the response given by the client when + he tries to authenticate. + Optional arguments: + entity_body - when 'qop' is set to 'auth-int' you MUST provide the + raw data you are going to send to the client (usually the + HTML page. + request_uri - the uri from the request line compared with the 'uri' + directive of the authorization map. They must represent + the same resource (unused at this time). + """ + + if auth_map['realm'] != kwargs.get('realm', None): + return False + + response = _computeDigestResponse(auth_map, password, method, A1,**kwargs) + + return response == auth_map["response"] + +def _checkBasicResponse (auth_map, password, method='GET', encrypt=None, **kwargs): + # Note that the Basic response doesn't provide the realm value so we cannot + # test it + try: + return encrypt(auth_map["password"], auth_map["username"]) == password + except TypeError: + return encrypt(auth_map["password"]) == password + +AUTH_RESPONSES = { + "basic": _checkBasicResponse, + "digest": _checkDigestResponse, +} + +def checkResponse (auth_map, password, method = "GET", encrypt=None, **kwargs): + """'checkResponse' compares the auth_map with the password and optionally + other arguments that each implementation might need. + + If the response is of type 'Basic' then the function has the following + signature:: + + checkBasicResponse (auth_map, password) -> bool + + If the response is of type 'Digest' then the function has the following + signature:: + + checkDigestResponse (auth_map, password, method = 'GET', A1 = None) -> bool + + The 'A1' argument is only used in MD5_SESS algorithm based responses. + Check md5SessionKey() for more info. + """ + checker = AUTH_RESPONSES[auth_map["auth_scheme"]] + return checker (auth_map, password, method=method, encrypt=encrypt, **kwargs) + + + + diff --git a/cherrypy/lib/httputil.py b/cherrypy/lib/httputil.py new file mode 100755 index 0000000..e005875 --- /dev/null +++ b/cherrypy/lib/httputil.py @@ -0,0 +1,469 @@ +"""HTTP library functions. + +This module contains functions for building an HTTP application +framework: any one, not just one whose name starts with "Ch". ;) If you +reference any modules from some popular framework inside *this* module, +FuManChu will personally hang you up by your thumbs and submit you +to a public caning. +""" + +from binascii import b2a_base64 +from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou, reversed, sorted +from cherrypy._cpcompat import basestring, iteritems, unicodestr, unquote_qs +response_codes = BaseHTTPRequestHandler.responses.copy() + +# From http://www.cherrypy.org/ticket/361 +response_codes[500] = ('Internal Server Error', + 'The server encountered an unexpected condition ' + 'which prevented it from fulfilling the request.') +response_codes[503] = ('Service Unavailable', + 'The server is currently unable to handle the ' + 'request due to a temporary overloading or ' + 'maintenance of the server.') + +import re +import urllib + + + +def urljoin(*atoms): + """Return the given path \*atoms, joined into a single URL. + + This will correctly join a SCRIPT_NAME and PATH_INFO into the + original URL, even if either atom is blank. + """ + url = "/".join([x for x in atoms if x]) + while "//" in url: + url = url.replace("//", "/") + # Special-case the final url of "", and return "/" instead. + return url or "/" + +def protocol_from_http(protocol_str): + """Return a protocol tuple from the given 'HTTP/x.y' string.""" + return int(protocol_str[5]), int(protocol_str[7]) + +def get_ranges(headervalue, content_length): + """Return a list of (start, stop) indices from a Range header, or None. + + Each (start, stop) tuple will be composed of two ints, which are suitable + for use in a slicing operation. That is, the header "Range: bytes=3-6", + if applied against a Python string, is requesting resource[3:7]. This + function will return the list [(3, 7)]. + + If this function returns an empty list, you should return HTTP 416. + """ + + if not headervalue: + return None + + result = [] + bytesunit, byteranges = headervalue.split("=", 1) + for brange in byteranges.split(","): + start, stop = [x.strip() for x in brange.split("-", 1)] + if start: + if not stop: + stop = content_length - 1 + start, stop = int(start), int(stop) + if start >= content_length: + # From rfc 2616 sec 14.16: + # "If the server receives a request (other than one + # including an If-Range request-header field) with an + # unsatisfiable Range request-header field (that is, + # all of whose byte-range-spec values have a first-byte-pos + # value greater than the current length of the selected + # resource), it SHOULD return a response code of 416 + # (Requested range not satisfiable)." + continue + if stop < start: + # From rfc 2616 sec 14.16: + # "If the server ignores a byte-range-spec because it + # is syntactically invalid, the server SHOULD treat + # the request as if the invalid Range header field + # did not exist. (Normally, this means return a 200 + # response containing the full entity)." + return None + result.append((start, stop + 1)) + else: + if not stop: + # See rfc quote above. + return None + # Negative subscript (last N bytes) + result.append((content_length - int(stop), content_length)) + + return result + + +class HeaderElement(object): + """An element (with parameters) from an HTTP header's element list.""" + + def __init__(self, value, params=None): + self.value = value + if params is None: + params = {} + self.params = params + + def __cmp__(self, other): + return cmp(self.value, other.value) + + def __str__(self): + p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] + return "%s%s" % (self.value, "".join(p)) + + def __unicode__(self): + return ntou(self.__str__()) + + def parse(elementstr): + """Transform 'token;key=val' to ('token', {'key': 'val'}).""" + # Split the element into a value and parameters. The 'value' may + # be of the form, "token=token", but we don't split that here. + atoms = [x.strip() for x in elementstr.split(";") if x.strip()] + if not atoms: + initial_value = '' + else: + initial_value = atoms.pop(0).strip() + params = {} + for atom in atoms: + atom = [x.strip() for x in atom.split("=", 1) if x.strip()] + key = atom.pop(0) + if atom: + val = atom[0] + else: + val = "" + params[key] = val + return initial_value, params + parse = staticmethod(parse) + + def from_str(cls, elementstr): + """Construct an instance from a string of the form 'token;key=val'.""" + ival, params = cls.parse(elementstr) + return cls(ival, params) + from_str = classmethod(from_str) + + +q_separator = re.compile(r'; *q *=') + +class AcceptElement(HeaderElement): + """An element (with parameters) from an Accept* header's element list. + + AcceptElement objects are comparable; the more-preferred object will be + "less than" the less-preferred object. They are also therefore sortable; + if you sort a list of AcceptElement objects, they will be listed in + priority order; the most preferred value will be first. Yes, it should + have been the other way around, but it's too late to fix now. + """ + + def from_str(cls, elementstr): + qvalue = None + # The first "q" parameter (if any) separates the initial + # media-range parameter(s) (if any) from the accept-params. + atoms = q_separator.split(elementstr, 1) + media_range = atoms.pop(0).strip() + if atoms: + # The qvalue for an Accept header can have extensions. The other + # headers cannot, but it's easier to parse them as if they did. + qvalue = HeaderElement.from_str(atoms[0].strip()) + + media_type, params = cls.parse(media_range) + if qvalue is not None: + params["q"] = qvalue + return cls(media_type, params) + from_str = classmethod(from_str) + + def qvalue(self): + val = self.params.get("q", "1") + if isinstance(val, HeaderElement): + val = val.value + return float(val) + qvalue = property(qvalue, doc="The qvalue, or priority, of this value.") + + def __cmp__(self, other): + diff = cmp(self.qvalue, other.qvalue) + if diff == 0: + diff = cmp(str(self), str(other)) + return diff + + +def header_elements(fieldname, fieldvalue): + """Return a sorted HeaderElement list from a comma-separated header string.""" + if not fieldvalue: + return [] + + result = [] + for element in fieldvalue.split(","): + if fieldname.startswith("Accept") or fieldname == 'TE': + hv = AcceptElement.from_str(element) + else: + hv = HeaderElement.from_str(element) + result.append(hv) + + return list(reversed(sorted(result))) + +def decode_TEXT(value): + r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> u"f\xfcr").""" + from email.Header import decode_header + atoms = decode_header(value) + decodedvalue = "" + for atom, charset in atoms: + if charset is not None: + atom = atom.decode(charset) + decodedvalue += atom + return decodedvalue + +def valid_status(status): + """Return legal HTTP status Code, Reason-phrase and Message. + + The status arg must be an int, or a str that begins with an int. + + If status is an int, or a str and no reason-phrase is supplied, + a default reason-phrase will be provided. + """ + + if not status: + status = 200 + + status = str(status) + parts = status.split(" ", 1) + if len(parts) == 1: + # No reason supplied. + code, = parts + reason = None + else: + code, reason = parts + reason = reason.strip() + + try: + code = int(code) + except ValueError: + raise ValueError("Illegal response status from server " + "(%s is non-numeric)." % repr(code)) + + if code < 100 or code > 599: + raise ValueError("Illegal response status from server " + "(%s is out of range)." % repr(code)) + + if code not in response_codes: + # code is unknown but not illegal + default_reason, message = "", "" + else: + default_reason, message = response_codes[code] + + if reason is None: + reason = default_reason + + return code, reason, message + + +def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): + """Parse a query given as a string argument. + + Arguments: + + qs: URL-encoded query string to be parsed + + keep_blank_values: flag indicating whether blank values in + URL encoded queries should be treated as blank strings. A + true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. + + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. + + Returns a dict, as G-d intended. + """ + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] + d = {} + for name_value in pairs: + if not name_value and not strict_parsing: + continue + nv = name_value.split('=', 1) + if len(nv) != 2: + if strict_parsing: + raise ValueError("bad query field: %r" % (name_value,)) + # Handle case of a control-name with no equal sign + if keep_blank_values: + nv.append('') + else: + continue + if len(nv[1]) or keep_blank_values: + name = unquote_qs(nv[0], encoding) + value = unquote_qs(nv[1], encoding) + if name in d: + if not isinstance(d[name], list): + d[name] = [d[name]] + d[name].append(value) + else: + d[name] = value + return d + + +image_map_pattern = re.compile(r"[0-9]+,[0-9]+") + +def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): + """Build a params dictionary from a query_string. + + Duplicate key/value pairs in the provided query_string will be + returned as {'key': [val1, val2, ...]}. Single key/values will + be returned as strings: {'key': 'value'}. + """ + if image_map_pattern.match(query_string): + # Server-side image map. Map the coords to 'x' and 'y' + # (like CGI::Request does). + pm = query_string.split(",") + pm = {'x': int(pm[0]), 'y': int(pm[1])} + else: + pm = _parse_qs(query_string, keep_blank_values, encoding=encoding) + return pm + + +class CaseInsensitiveDict(dict): + """A case-insensitive dict subclass. + + Each key is changed on entry to str(key).title(). + """ + + def __getitem__(self, key): + return dict.__getitem__(self, str(key).title()) + + def __setitem__(self, key, value): + dict.__setitem__(self, str(key).title(), value) + + def __delitem__(self, key): + dict.__delitem__(self, str(key).title()) + + def __contains__(self, key): + return dict.__contains__(self, str(key).title()) + + def get(self, key, default=None): + return dict.get(self, str(key).title(), default) + + def has_key(self, key): + return dict.has_key(self, str(key).title()) + + def update(self, E): + for k in E.keys(): + self[str(k).title()] = E[k] + + def fromkeys(cls, seq, value=None): + newdict = cls() + for k in seq: + newdict[str(k).title()] = value + return newdict + fromkeys = classmethod(fromkeys) + + def setdefault(self, key, x=None): + key = str(key).title() + try: + return self[key] + except KeyError: + self[key] = x + return x + + def pop(self, key, default): + return dict.pop(self, str(key).title(), default) + + +# TEXT = <any OCTET except CTLs, but including LWS> +# +# A CRLF is allowed in the definition of TEXT only as part of a header +# field continuation. It is expected that the folding LWS will be +# replaced with a single SP before interpretation of the TEXT value." +header_translate_table = ''.join([chr(i) for i in xrange(256)]) +header_translate_deletechars = ''.join([chr(i) for i in xrange(32)]) + chr(127) + + +class HeaderMap(CaseInsensitiveDict): + """A dict subclass for HTTP request and response headers. + + Each key is changed on entry to str(key).title(). This allows headers + to be case-insensitive and avoid duplicates. + + Values are header values (decoded according to :rfc:`2047` if necessary). + """ + + protocol=(1, 1) + encodings = ["ISO-8859-1"] + + # Someday, when http-bis is done, this will probably get dropped + # since few servers, clients, or intermediaries do it. But until then, + # we're going to obey the spec as is. + # "Words of *TEXT MAY contain characters from character sets other than + # ISO-8859-1 only when encoded according to the rules of RFC 2047." + use_rfc_2047 = True + + def elements(self, key): + """Return a sorted list of HeaderElements for the given header.""" + key = str(key).title() + value = self.get(key) + return header_elements(key, value) + + def values(self, key): + """Return a sorted list of HeaderElement.value for the given header.""" + return [e.value for e in self.elements(key)] + + def output(self): + """Transform self into a list of (name, value) tuples.""" + header_list = [] + for k, v in self.items(): + if isinstance(k, unicodestr): + k = self.encode(k) + + if not isinstance(v, basestring): + v = str(v) + + if isinstance(v, unicodestr): + v = self.encode(v) + + # See header_translate_* constants above. + # Replace only if you really know what you're doing. + k = k.translate(header_translate_table, header_translate_deletechars) + v = v.translate(header_translate_table, header_translate_deletechars) + + header_list.append((k, v)) + return header_list + + def encode(self, v): + """Return the given header name or value, encoded for HTTP output.""" + for enc in self.encodings: + try: + return v.encode(enc) + except UnicodeEncodeError: + continue + + if self.protocol == (1, 1) and self.use_rfc_2047: + # Encode RFC-2047 TEXT + # (e.g. u"\u8200" -> "=?utf-8?b?6IiA?="). + # We do our own here instead of using the email module + # because we never want to fold lines--folding has + # been deprecated by the HTTP working group. + v = b2a_base64(v.encode('utf-8')) + return (ntob('=?utf-8?b?') + v.strip(ntob('\n')) + ntob('?=')) + + raise ValueError("Could not encode header part %r using " + "any of the encodings %r." % + (v, self.encodings)) + + +class Host(object): + """An internet address. + + name + Should be the client's host name. If not available (because no DNS + lookup is performed), the IP address should be used instead. + + """ + + ip = "0.0.0.0" + port = 80 + name = "unknown.tld" + + def __init__(self, ip, port, name=None): + self.ip = ip + self.port = port + if name is None: + name = ip + self.name = name + + def __repr__(self): + return "httputil.Host(%r, %r, %r)" % (self.ip, self.port, self.name) diff --git a/cherrypy/lib/jsontools.py b/cherrypy/lib/jsontools.py new file mode 100755 index 0000000..09042e4 --- /dev/null +++ b/cherrypy/lib/jsontools.py @@ -0,0 +1,87 @@ +import sys +import cherrypy +from cherrypy._cpcompat import basestring, ntou, json, json_encode, json_decode + +def json_processor(entity): + """Read application/json data into request.json.""" + if not entity.headers.get(ntou("Content-Length"), ntou("")): + raise cherrypy.HTTPError(411) + + body = entity.fp.read() + try: + cherrypy.serving.request.json = json_decode(body.decode('utf-8')) + except ValueError: + raise cherrypy.HTTPError(400, 'Invalid JSON document') + +def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], + force=True, debug=False, processor = json_processor): + """Add a processor to parse JSON request entities: + The default processor places the parsed data into request.json. + + Incoming request entities which match the given content_type(s) will + be deserialized from JSON to the Python equivalent, and the result + stored at cherrypy.request.json. The 'content_type' argument may + be a Content-Type string or a list of allowable Content-Type strings. + + If the 'force' argument is True (the default), then entities of other + content types will not be allowed; "415 Unsupported Media Type" is + raised instead. + + Supply your own processor to use a custom decoder, or to handle the parsed + data differently. The processor can be configured via + tools.json_in.processor or via the decorator method. + + Note that the deserializer requires the client send a Content-Length + request header, or it will raise "411 Length Required". If for any + other reason the request entity cannot be deserialized from JSON, + it will raise "400 Bad Request: Invalid JSON document". + + You must be using Python 2.6 or greater, or have the 'simplejson' + package importable; otherwise, ValueError is raised during processing. + """ + request = cherrypy.serving.request + if isinstance(content_type, basestring): + content_type = [content_type] + + if force: + if debug: + cherrypy.log('Removing body processors %s' % + repr(request.body.processors.keys()), 'TOOLS.JSON_IN') + request.body.processors.clear() + request.body.default_proc = cherrypy.HTTPError( + 415, 'Expected an entity of content type %s' % + ', '.join(content_type)) + + for ct in content_type: + if debug: + cherrypy.log('Adding body processor for %s' % ct, 'TOOLS.JSON_IN') + request.body.processors[ct] = processor + +def json_handler(*args, **kwargs): + value = cherrypy.serving.request._json_inner_handler(*args, **kwargs) + return json_encode(value) + +def json_out(content_type='application/json', debug=False, handler=json_handler): + """Wrap request.handler to serialize its output to JSON. Sets Content-Type. + + If the given content_type is None, the Content-Type response header + is not set. + + Provide your own handler to use a custom encoder. For example + cherrypy.config['tools.json_out.handler'] = <function>, or + @json_out(handler=function). + + You must be using Python 2.6 or greater, or have the 'simplejson' + package importable; otherwise, ValueError is raised during processing. + """ + request = cherrypy.serving.request + if debug: + cherrypy.log('Replacing %s with JSON handler' % request.handler, + 'TOOLS.JSON_OUT') + request._json_inner_handler = request.handler + request.handler = handler + if content_type is not None: + if debug: + cherrypy.log('Setting Content-Type to %s' % ct, 'TOOLS.JSON_OUT') + cherrypy.serving.response.headers['Content-Type'] = content_type + diff --git a/cherrypy/lib/profiler.py b/cherrypy/lib/profiler.py new file mode 100755 index 0000000..785d58a --- /dev/null +++ b/cherrypy/lib/profiler.py @@ -0,0 +1,208 @@ +"""Profiler tools for CherryPy. + +CherryPy users +============== + +You can profile any of your pages as follows:: + + from cherrypy.lib import profiler + + class Root: + p = profile.Profiler("/path/to/profile/dir") + + def index(self): + self.p.run(self._index) + index.exposed = True + + def _index(self): + return "Hello, world!" + + cherrypy.tree.mount(Root()) + +You can also turn on profiling for all requests +using the ``make_app`` function as WSGI middleware. + +CherryPy developers +=================== + +This module can be used whenever you make changes to CherryPy, +to get a quick sanity-check on overall CP performance. Use the +``--profile`` flag when running the test suite. Then, use the ``serve()`` +function to browse the results in a web browser. If you run this +module from the command line, it will call ``serve()`` for you. + +""" + + +def new_func_strip_path(func_name): + """Make profiler output more readable by adding ``__init__`` modules' parents""" + filename, line, name = func_name + if filename.endswith("__init__.py"): + return os.path.basename(filename[:-12]) + filename[-12:], line, name + return os.path.basename(filename), line, name + +try: + import profile + import pstats + pstats.func_strip_path = new_func_strip_path +except ImportError: + profile = None + pstats = None + +import os, os.path +import sys +import warnings + +from cherrypy._cpcompat import BytesIO + +_count = 0 + +class Profiler(object): + + def __init__(self, path=None): + if not path: + path = os.path.join(os.path.dirname(__file__), "profile") + self.path = path + if not os.path.exists(path): + os.makedirs(path) + + def run(self, func, *args, **params): + """Dump profile data into self.path.""" + global _count + c = _count = _count + 1 + path = os.path.join(self.path, "cp_%04d.prof" % c) + prof = profile.Profile() + result = prof.runcall(func, *args, **params) + prof.dump_stats(path) + return result + + def statfiles(self): + """:rtype: list of available profiles. + """ + return [f for f in os.listdir(self.path) + if f.startswith("cp_") and f.endswith(".prof")] + + def stats(self, filename, sortby='cumulative'): + """:rtype stats(index): output of print_stats() for the given profile. + """ + sio = BytesIO() + if sys.version_info >= (2, 5): + s = pstats.Stats(os.path.join(self.path, filename), stream=sio) + s.strip_dirs() + s.sort_stats(sortby) + s.print_stats() + else: + # pstats.Stats before Python 2.5 didn't take a 'stream' arg, + # but just printed to stdout. So re-route stdout. + s = pstats.Stats(os.path.join(self.path, filename)) + s.strip_dirs() + s.sort_stats(sortby) + oldout = sys.stdout + try: + sys.stdout = sio + s.print_stats() + finally: + sys.stdout = oldout + response = sio.getvalue() + sio.close() + return response + + def index(self): + return """<html> + <head><title>CherryPy profile data</title></head> + <frameset cols='200, 1*'> + <frame src='menu' /> + <frame name='main' src='' /> + </frameset> + </html> + """ + index.exposed = True + + def menu(self): + yield "<h2>Profiling runs</h2>" + yield "<p>Click on one of the runs below to see profiling data.</p>" + runs = self.statfiles() + runs.sort() + for i in runs: + yield "<a href='report?filename=%s' target='main'>%s</a><br />" % (i, i) + menu.exposed = True + + def report(self, filename): + import cherrypy + cherrypy.response.headers['Content-Type'] = 'text/plain' + return self.stats(filename) + report.exposed = True + + +class ProfileAggregator(Profiler): + + def __init__(self, path=None): + Profiler.__init__(self, path) + global _count + self.count = _count = _count + 1 + self.profiler = profile.Profile() + + def run(self, func, *args): + path = os.path.join(self.path, "cp_%04d.prof" % self.count) + result = self.profiler.runcall(func, *args) + self.profiler.dump_stats(path) + return result + + +class make_app: + def __init__(self, nextapp, path=None, aggregate=False): + """Make a WSGI middleware app which wraps 'nextapp' with profiling. + + nextapp + the WSGI application to wrap, usually an instance of + cherrypy.Application. + + path + where to dump the profiling output. + + aggregate + if True, profile data for all HTTP requests will go in + a single file. If False (the default), each HTTP request will + dump its profile data into a separate file. + + """ + if profile is None or pstats is None: + msg = ("Your installation of Python does not have a profile module. " + "If you're on Debian, try `sudo apt-get install python-profiler`. " + "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") + warnings.warn(msg) + + self.nextapp = nextapp + self.aggregate = aggregate + if aggregate: + self.profiler = ProfileAggregator(path) + else: + self.profiler = Profiler(path) + + def __call__(self, environ, start_response): + def gather(): + result = [] + for line in self.nextapp(environ, start_response): + result.append(line) + return result + return self.profiler.run(gather) + + +def serve(path=None, port=8080): + if profile is None or pstats is None: + msg = ("Your installation of Python does not have a profile module. " + "If you're on Debian, try `sudo apt-get install python-profiler`. " + "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") + warnings.warn(msg) + + import cherrypy + cherrypy.config.update({'server.socket_port': int(port), + 'server.thread_pool': 10, + 'environment': "production", + }) + cherrypy.quickstart(Profiler(path)) + + +if __name__ == "__main__": + serve(*tuple(sys.argv[1:])) + diff --git a/cherrypy/lib/reprconf.py b/cherrypy/lib/reprconf.py new file mode 100755 index 0000000..e18949e --- /dev/null +++ b/cherrypy/lib/reprconf.py @@ -0,0 +1,351 @@ +"""Generic configuration system using unrepr. + +Configuration data may be supplied as a Python dictionary, as a filename, +or as an open file object. When you supply a filename or file, Python's +builtin ConfigParser is used (with some extensions). + +Namespaces +---------- + +Configuration keys are separated into namespaces by the first "." in the key. + +The only key that cannot exist in a namespace is the "environment" entry. +This special entry 'imports' other config entries from a template stored in +the Config.environments dict. + +You can define your own namespaces to be called when new config is merged +by adding a named handler to Config.namespaces. The name can be any string, +and the handler must be either a callable or a context manager. +""" + +try: + # Python 3.0+ + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser + +try: + set +except NameError: + from sets import Set as set +import sys + +def as_dict(config): + """Return a dict from 'config' whether it is a dict, file, or filename.""" + if isinstance(config, basestring): + config = Parser().dict_from_file(config) + elif hasattr(config, 'read'): + config = Parser().dict_from_file(config) + return config + + +class NamespaceSet(dict): + """A dict of config namespace names and handlers. + + Each config entry should begin with a namespace name; the corresponding + namespace handler will be called once for each config entry in that + namespace, and will be passed two arguments: the config key (with the + namespace removed) and the config value. + + Namespace handlers may be any Python callable; they may also be + Python 2.5-style 'context managers', in which case their __enter__ + method should return a callable to be used as the handler. + See cherrypy.tools (the Toolbox class) for an example. + """ + + def __call__(self, config): + """Iterate through config and pass it to each namespace handler. + + config + A flat dict, where keys use dots to separate + namespaces, and values are arbitrary. + + The first name in each config key is used to look up the corresponding + namespace handler. For example, a config entry of {'tools.gzip.on': v} + will call the 'tools' namespace handler with the args: ('gzip.on', v) + """ + # Separate the given config into namespaces + ns_confs = {} + for k in config: + if "." in k: + ns, name = k.split(".", 1) + bucket = ns_confs.setdefault(ns, {}) + bucket[name] = config[k] + + # I chose __enter__ and __exit__ so someday this could be + # rewritten using Python 2.5's 'with' statement: + # for ns, handler in self.iteritems(): + # with handler as callable: + # for k, v in ns_confs.get(ns, {}).iteritems(): + # callable(k, v) + for ns, handler in self.items(): + exit = getattr(handler, "__exit__", None) + if exit: + callable = handler.__enter__() + no_exc = True + try: + try: + for k, v in ns_confs.get(ns, {}).items(): + callable(k, v) + except: + # The exceptional case is handled here + no_exc = False + if exit is None: + raise + if not exit(*sys.exc_info()): + raise + # The exception is swallowed if exit() returns true + finally: + # The normal and non-local-goto cases are handled here + if no_exc and exit: + exit(None, None, None) + else: + for k, v in ns_confs.get(ns, {}).items(): + handler(k, v) + + def __repr__(self): + return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, + dict.__repr__(self)) + + def __copy__(self): + newobj = self.__class__() + newobj.update(self) + return newobj + copy = __copy__ + + +class Config(dict): + """A dict-like set of configuration data, with defaults and namespaces. + + May take a file, filename, or dict. + """ + + defaults = {} + environments = {} + namespaces = NamespaceSet() + + def __init__(self, file=None, **kwargs): + self.reset() + if file is not None: + self.update(file) + if kwargs: + self.update(kwargs) + + def reset(self): + """Reset self to default values.""" + self.clear() + dict.update(self, self.defaults) + + def update(self, config): + """Update self from a dict, file or filename.""" + if isinstance(config, basestring): + # Filename + config = Parser().dict_from_file(config) + elif hasattr(config, 'read'): + # Open file object + config = Parser().dict_from_file(config) + else: + config = config.copy() + self._apply(config) + + def _apply(self, config): + """Update self from a dict.""" + which_env = config.get('environment') + if which_env: + env = self.environments[which_env] + for k in env: + if k not in config: + config[k] = env[k] + + dict.update(self, config) + self.namespaces(config) + + def __setitem__(self, k, v): + dict.__setitem__(self, k, v) + self.namespaces({k: v}) + + +class Parser(ConfigParser): + """Sub-class of ConfigParser that keeps the case of options and that + raises an exception if the file cannot be read. + """ + + def optionxform(self, optionstr): + return optionstr + + def read(self, filenames): + if isinstance(filenames, basestring): + filenames = [filenames] + for filename in filenames: + # try: + # fp = open(filename) + # except IOError: + # continue + fp = open(filename) + try: + self._read(fp, filename) + finally: + fp.close() + + def as_dict(self, raw=False, vars=None): + """Convert an INI file to a dictionary""" + # Load INI file into a dict + result = {} + for section in self.sections(): + if section not in result: + result[section] = {} + for option in self.options(section): + value = self.get(section, option, raw, vars) + try: + value = unrepr(value) + except Exception, x: + msg = ("Config error in section: %r, option: %r, " + "value: %r. Config values must be valid Python." % + (section, option, value)) + raise ValueError(msg, x.__class__.__name__, x.args) + result[section][option] = value + return result + + def dict_from_file(self, file): + if hasattr(file, 'read'): + self.readfp(file) + else: + self.read(file) + return self.as_dict() + + +# public domain "unrepr" implementation, found on the web and then improved. + +class _Builder: + + def build(self, o): + m = getattr(self, 'build_' + o.__class__.__name__, None) + if m is None: + raise TypeError("unrepr does not recognize %s" % + repr(o.__class__.__name__)) + return m(o) + + def build_Subscript(self, o): + expr, flags, subs = o.getChildren() + expr = self.build(expr) + subs = self.build(subs) + return expr[subs] + + def build_CallFunc(self, o): + children = map(self.build, o.getChildren()) + callee = children.pop(0) + kwargs = children.pop() or {} + starargs = children.pop() or () + args = tuple(children) + tuple(starargs) + return callee(*args, **kwargs) + + def build_List(self, o): + return map(self.build, o.getChildren()) + + def build_Const(self, o): + return o.value + + def build_Dict(self, o): + d = {} + i = iter(map(self.build, o.getChildren())) + for el in i: + d[el] = i.next() + return d + + def build_Tuple(self, o): + return tuple(self.build_List(o)) + + def build_Name(self, o): + name = o.name + if name == 'None': + return None + if name == 'True': + return True + if name == 'False': + return False + + # See if the Name is a package or module. If it is, import it. + try: + return modules(name) + except ImportError: + pass + + # See if the Name is in builtins. + try: + import __builtin__ + return getattr(__builtin__, name) + except AttributeError: + pass + + raise TypeError("unrepr could not resolve the name %s" % repr(name)) + + def build_Add(self, o): + left, right = map(self.build, o.getChildren()) + return left + right + + def build_Getattr(self, o): + parent = self.build(o.expr) + return getattr(parent, o.attrname) + + def build_NoneType(self, o): + return None + + def build_UnarySub(self, o): + return -self.build(o.getChildren()[0]) + + def build_UnaryAdd(self, o): + return self.build(o.getChildren()[0]) + + +def _astnode(s): + """Return a Python ast Node compiled from a string.""" + try: + import compiler + except ImportError: + # Fallback to eval when compiler package is not available, + # e.g. IronPython 1.0. + return eval(s) + + p = compiler.parse("__tempvalue__ = " + s) + return p.getChildren()[1].getChildren()[0].getChildren()[1] + + +def unrepr(s): + """Return a Python object compiled from a string.""" + if not s: + return s + obj = _astnode(s) + return _Builder().build(obj) + + +def modules(modulePath): + """Load a module and retrieve a reference to that module.""" + try: + mod = sys.modules[modulePath] + if mod is None: + raise KeyError() + except KeyError: + # The last [''] is important. + mod = __import__(modulePath, globals(), locals(), ['']) + return mod + +def attributes(full_attribute_name): + """Load a module and retrieve an attribute of that module.""" + + # Parse out the path, module, and attribute + last_dot = full_attribute_name.rfind(".") + attr_name = full_attribute_name[last_dot + 1:] + mod_path = full_attribute_name[:last_dot] + + mod = modules(mod_path) + # Let an AttributeError propagate outward. + try: + attr = getattr(mod, attr_name) + except AttributeError: + raise AttributeError("'%s' object has no attribute '%s'" + % (mod_path, attr_name)) + + # Return a reference to the attribute. + return attr + + diff --git a/cherrypy/lib/sessions.py b/cherrypy/lib/sessions.py new file mode 100755 index 0000000..42c2800 --- /dev/null +++ b/cherrypy/lib/sessions.py @@ -0,0 +1,832 @@ +"""Session implementation for CherryPy. + +You need to edit your config file to use sessions. Here's an example:: + + [/] + tools.sessions.on = True + tools.sessions.storage_type = "file" + tools.sessions.storage_path = "/home/site/sessions" + tools.sessions.timeout = 60 + +This sets the session to be stored in files in the directory /home/site/sessions, +and the session timeout to 60 minutes. If you omit ``storage_type`` the sessions +will be saved in RAM. ``tools.sessions.on`` is the only required line for +working sessions, the rest are optional. + +By default, the session ID is passed in a cookie, so the client's browser must +have cookies enabled for your site. + +To set data for the current session, use +``cherrypy.session['fieldname'] = 'fieldvalue'``; +to get data use ``cherrypy.session.get('fieldname')``. + +================ +Locking sessions +================ + +By default, the ``'locking'`` mode of sessions is ``'implicit'``, which means +the session is locked early and unlocked late. If you want to control when the +session data is locked and unlocked, set ``tools.sessions.locking = 'explicit'``. +Then call ``cherrypy.session.acquire_lock()`` and ``cherrypy.session.release_lock()``. +Regardless of which mode you use, the session is guaranteed to be unlocked when +the request is complete. + +================= +Expiring Sessions +================= + +You can force a session to expire with :func:`cherrypy.lib.sessions.expire`. +Simply call that function at the point you want the session to expire, and it +will cause the session cookie to expire client-side. + +=========================== +Session Fixation Protection +=========================== + +If CherryPy receives, via a request cookie, a session id that it does not +recognize, it will reject that id and create a new one to return in the +response cookie. This `helps prevent session fixation attacks +<http://en.wikipedia.org/wiki/Session_fixation#Regenerate_SID_on_each_request>`_. +However, CherryPy "recognizes" a session id by looking up the saved session +data for that id. Therefore, if you never save any session data, +**you will get a new session id for every request**. + +================ +Sharing Sessions +================ + +If you run multiple instances of CherryPy (for example via mod_python behind +Apache prefork), you most likely cannot use the RAM session backend, since each +instance of CherryPy will have its own memory space. Use a different backend +instead, and verify that all instances are pointing at the same file or db +location. Alternately, you might try a load balancer which makes sessions +"sticky". Google is your friend, there. + +================ +Expiration Dates +================ + +The response cookie will possess an expiration date to inform the client at +which point to stop sending the cookie back in requests. If the server time +and client time differ, expect sessions to be unreliable. **Make sure the +system time of your server is accurate**. + +CherryPy defaults to a 60-minute session timeout, which also applies to the +cookie which is sent to the client. Unfortunately, some versions of Safari +("4 public beta" on Windows XP at least) appear to have a bug in their parsing +of the GMT expiration date--they appear to interpret the date as one hour in +the past. Sixty minutes minus one hour is pretty close to zero, so you may +experience this bug as a new session id for every request, unless the requests +are less than one second apart. To fix, try increasing the session.timeout. + +On the other extreme, some users report Firefox sending cookies after their +expiration date, although this was on a system with an inaccurate system time. +Maybe FF doesn't trust system time. +""" + +import datetime +import os +import random +import time +import threading +import types +from warnings import warn + +import cherrypy +from cherrypy._cpcompat import copyitems, pickle, random20 +from cherrypy.lib import httputil + + +missing = object() + +class Session(object): + """A CherryPy dict-like Session object (one per request).""" + + _id = None + + id_observers = None + "A list of callbacks to which to pass new id's." + + def _get_id(self): + return self._id + def _set_id(self, value): + self._id = value + for o in self.id_observers: + o(value) + id = property(_get_id, _set_id, doc="The current session ID.") + + timeout = 60 + "Number of minutes after which to delete session data." + + locked = False + """ + If True, this session instance has exclusive read/write access + to session data.""" + + loaded = False + """ + If True, data has been retrieved from storage. This should happen + automatically on the first attempt to access session data.""" + + clean_thread = None + "Class-level Monitor which calls self.clean_up." + + clean_freq = 5 + "The poll rate for expired session cleanup in minutes." + + originalid = None + "The session id passed by the client. May be missing or unsafe." + + missing = False + "True if the session requested by the client did not exist." + + regenerated = False + """ + True if the application called session.regenerate(). This is not set by + internal calls to regenerate the session id.""" + + debug=False + + def __init__(self, id=None, **kwargs): + self.id_observers = [] + self._data = {} + + for k, v in kwargs.items(): + setattr(self, k, v) + + self.originalid = id + self.missing = False + if id is None: + if self.debug: + cherrypy.log('No id given; making a new one', 'TOOLS.SESSIONS') + self._regenerate() + else: + self.id = id + if not self._exists(): + if self.debug: + cherrypy.log('Expired or malicious session %r; ' + 'making a new one' % id, 'TOOLS.SESSIONS') + # Expired or malicious session. Make a new one. + # See http://www.cherrypy.org/ticket/709. + self.id = None + self.missing = True + self._regenerate() + + def regenerate(self): + """Replace the current session (with a new id).""" + self.regenerated = True + self._regenerate() + + def _regenerate(self): + if self.id is not None: + self.delete() + + old_session_was_locked = self.locked + if old_session_was_locked: + self.release_lock() + + self.id = None + while self.id is None: + self.id = self.generate_id() + # Assert that the generated id is not already stored. + if self._exists(): + self.id = None + + if old_session_was_locked: + self.acquire_lock() + + def clean_up(self): + """Clean up expired sessions.""" + pass + + def generate_id(self): + """Return a new session id.""" + return random20() + + def save(self): + """Save session data.""" + try: + # If session data has never been loaded then it's never been + # accessed: no need to save it + if self.loaded: + t = datetime.timedelta(seconds = self.timeout * 60) + expiration_time = datetime.datetime.now() + t + if self.debug: + cherrypy.log('Saving with expiry %s' % expiration_time, + 'TOOLS.SESSIONS') + self._save(expiration_time) + + finally: + if self.locked: + # Always release the lock if the user didn't release it + self.release_lock() + + def load(self): + """Copy stored session data into this session instance.""" + data = self._load() + # data is either None or a tuple (session_data, expiration_time) + if data is None or data[1] < datetime.datetime.now(): + if self.debug: + cherrypy.log('Expired session, flushing data', 'TOOLS.SESSIONS') + self._data = {} + else: + self._data = data[0] + self.loaded = True + + # Stick the clean_thread in the class, not the instance. + # The instances are created and destroyed per-request. + cls = self.__class__ + if self.clean_freq and not cls.clean_thread: + # clean_up is in instancemethod and not a classmethod, + # so that tool config can be accessed inside the method. + t = cherrypy.process.plugins.Monitor( + cherrypy.engine, self.clean_up, self.clean_freq * 60, + name='Session cleanup') + t.subscribe() + cls.clean_thread = t + t.start() + + def delete(self): + """Delete stored session data.""" + self._delete() + + def __getitem__(self, key): + if not self.loaded: self.load() + return self._data[key] + + def __setitem__(self, key, value): + if not self.loaded: self.load() + self._data[key] = value + + def __delitem__(self, key): + if not self.loaded: self.load() + del self._data[key] + + def pop(self, key, default=missing): + """Remove the specified key and return the corresponding value. + If key is not found, default is returned if given, + otherwise KeyError is raised. + """ + if not self.loaded: self.load() + if default is missing: + return self._data.pop(key) + else: + return self._data.pop(key, default) + + def __contains__(self, key): + if not self.loaded: self.load() + return key in self._data + + def has_key(self, key): + """D.has_key(k) -> True if D has a key k, else False.""" + if not self.loaded: self.load() + return key in self._data + + def get(self, key, default=None): + """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" + if not self.loaded: self.load() + return self._data.get(key, default) + + def update(self, d): + """D.update(E) -> None. Update D from E: for k in E: D[k] = E[k].""" + if not self.loaded: self.load() + self._data.update(d) + + def setdefault(self, key, default=None): + """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D.""" + if not self.loaded: self.load() + return self._data.setdefault(key, default) + + def clear(self): + """D.clear() -> None. Remove all items from D.""" + if not self.loaded: self.load() + self._data.clear() + + def keys(self): + """D.keys() -> list of D's keys.""" + if not self.loaded: self.load() + return self._data.keys() + + def items(self): + """D.items() -> list of D's (key, value) pairs, as 2-tuples.""" + if not self.loaded: self.load() + return self._data.items() + + def values(self): + """D.values() -> list of D's values.""" + if not self.loaded: self.load() + return self._data.values() + + +class RamSession(Session): + + # Class-level objects. Don't rebind these! + cache = {} + locks = {} + + def clean_up(self): + """Clean up expired sessions.""" + now = datetime.datetime.now() + for id, (data, expiration_time) in copyitems(self.cache): + if expiration_time <= now: + try: + del self.cache[id] + except KeyError: + pass + try: + del self.locks[id] + except KeyError: + pass + + def _exists(self): + return self.id in self.cache + + def _load(self): + return self.cache.get(self.id) + + def _save(self, expiration_time): + self.cache[self.id] = (self._data, expiration_time) + + def _delete(self): + self.cache.pop(self.id, None) + + def acquire_lock(self): + """Acquire an exclusive lock on the currently-loaded session data.""" + self.locked = True + self.locks.setdefault(self.id, threading.RLock()).acquire() + + def release_lock(self): + """Release the lock on the currently-loaded session data.""" + self.locks[self.id].release() + self.locked = False + + def __len__(self): + """Return the number of active sessions.""" + return len(self.cache) + + +class FileSession(Session): + """Implementation of the File backend for sessions + + storage_path + The folder where session data will be saved. Each session + will be saved as pickle.dump(data, expiration_time) in its own file; + the filename will be self.SESSION_PREFIX + self.id. + + """ + + SESSION_PREFIX = 'session-' + LOCK_SUFFIX = '.lock' + pickle_protocol = pickle.HIGHEST_PROTOCOL + + def __init__(self, id=None, **kwargs): + # The 'storage_path' arg is required for file-based sessions. + kwargs['storage_path'] = os.path.abspath(kwargs['storage_path']) + Session.__init__(self, id=id, **kwargs) + + def setup(cls, **kwargs): + """Set up the storage system for file-based sessions. + + This should only be called once per process; this will be done + automatically when using sessions.init (as the built-in Tool does). + """ + # The 'storage_path' arg is required for file-based sessions. + kwargs['storage_path'] = os.path.abspath(kwargs['storage_path']) + + for k, v in kwargs.items(): + setattr(cls, k, v) + + # Warn if any lock files exist at startup. + lockfiles = [fname for fname in os.listdir(cls.storage_path) + if (fname.startswith(cls.SESSION_PREFIX) + and fname.endswith(cls.LOCK_SUFFIX))] + if lockfiles: + plural = ('', 's')[len(lockfiles) > 1] + warn("%s session lockfile%s found at startup. If you are " + "only running one process, then you may need to " + "manually delete the lockfiles found at %r." + % (len(lockfiles), plural, cls.storage_path)) + setup = classmethod(setup) + + def _get_file_path(self): + f = os.path.join(self.storage_path, self.SESSION_PREFIX + self.id) + if not os.path.abspath(f).startswith(self.storage_path): + raise cherrypy.HTTPError(400, "Invalid session id in cookie.") + return f + + def _exists(self): + path = self._get_file_path() + return os.path.exists(path) + + def _load(self, path=None): + if path is None: + path = self._get_file_path() + try: + f = open(path, "rb") + try: + return pickle.load(f) + finally: + f.close() + except (IOError, EOFError): + return None + + def _save(self, expiration_time): + f = open(self._get_file_path(), "wb") + try: + pickle.dump((self._data, expiration_time), f, self.pickle_protocol) + finally: + f.close() + + def _delete(self): + try: + os.unlink(self._get_file_path()) + except OSError: + pass + + def acquire_lock(self, path=None): + """Acquire an exclusive lock on the currently-loaded session data.""" + if path is None: + path = self._get_file_path() + path += self.LOCK_SUFFIX + while True: + try: + lockfd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL) + except OSError: + time.sleep(0.1) + else: + os.close(lockfd) + break + self.locked = True + + def release_lock(self, path=None): + """Release the lock on the currently-loaded session data.""" + if path is None: + path = self._get_file_path() + os.unlink(path + self.LOCK_SUFFIX) + self.locked = False + + def clean_up(self): + """Clean up expired sessions.""" + now = datetime.datetime.now() + # Iterate over all session files in self.storage_path + for fname in os.listdir(self.storage_path): + if (fname.startswith(self.SESSION_PREFIX) + and not fname.endswith(self.LOCK_SUFFIX)): + # We have a session file: lock and load it and check + # if it's expired. If it fails, nevermind. + path = os.path.join(self.storage_path, fname) + self.acquire_lock(path) + try: + contents = self._load(path) + # _load returns None on IOError + if contents is not None: + data, expiration_time = contents + if expiration_time < now: + # Session expired: deleting it + os.unlink(path) + finally: + self.release_lock(path) + + def __len__(self): + """Return the number of active sessions.""" + return len([fname for fname in os.listdir(self.storage_path) + if (fname.startswith(self.SESSION_PREFIX) + and not fname.endswith(self.LOCK_SUFFIX))]) + + +class PostgresqlSession(Session): + """ Implementation of the PostgreSQL backend for sessions. It assumes + a table like this:: + + create table session ( + id varchar(40), + data text, + expiration_time timestamp + ) + + You must provide your own get_db function. + """ + + pickle_protocol = pickle.HIGHEST_PROTOCOL + + def __init__(self, id=None, **kwargs): + Session.__init__(self, id, **kwargs) + self.cursor = self.db.cursor() + + def setup(cls, **kwargs): + """Set up the storage system for Postgres-based sessions. + + This should only be called once per process; this will be done + automatically when using sessions.init (as the built-in Tool does). + """ + for k, v in kwargs.items(): + setattr(cls, k, v) + + self.db = self.get_db() + setup = classmethod(setup) + + def __del__(self): + if self.cursor: + self.cursor.close() + self.db.commit() + + def _exists(self): + # Select session data from table + self.cursor.execute('select data, expiration_time from session ' + 'where id=%s', (self.id,)) + rows = self.cursor.fetchall() + return bool(rows) + + def _load(self): + # Select session data from table + self.cursor.execute('select data, expiration_time from session ' + 'where id=%s', (self.id,)) + rows = self.cursor.fetchall() + if not rows: + return None + + pickled_data, expiration_time = rows[0] + data = pickle.loads(pickled_data) + return data, expiration_time + + def _save(self, expiration_time): + pickled_data = pickle.dumps(self._data, self.pickle_protocol) + self.cursor.execute('update session set data = %s, ' + 'expiration_time = %s where id = %s', + (pickled_data, expiration_time, self.id)) + + def _delete(self): + self.cursor.execute('delete from session where id=%s', (self.id,)) + + def acquire_lock(self): + """Acquire an exclusive lock on the currently-loaded session data.""" + # We use the "for update" clause to lock the row + self.locked = True + self.cursor.execute('select id from session where id=%s for update', + (self.id,)) + + def release_lock(self): + """Release the lock on the currently-loaded session data.""" + # We just close the cursor and that will remove the lock + # introduced by the "for update" clause + self.cursor.close() + self.locked = False + + def clean_up(self): + """Clean up expired sessions.""" + self.cursor.execute('delete from session where expiration_time < %s', + (datetime.datetime.now(),)) + + +class MemcachedSession(Session): + + # The most popular memcached client for Python isn't thread-safe. + # Wrap all .get and .set operations in a single lock. + mc_lock = threading.RLock() + + # This is a seperate set of locks per session id. + locks = {} + + servers = ['127.0.0.1:11211'] + + def setup(cls, **kwargs): + """Set up the storage system for memcached-based sessions. + + This should only be called once per process; this will be done + automatically when using sessions.init (as the built-in Tool does). + """ + for k, v in kwargs.items(): + setattr(cls, k, v) + + import memcache + cls.cache = memcache.Client(cls.servers) + setup = classmethod(setup) + + def _exists(self): + self.mc_lock.acquire() + try: + return bool(self.cache.get(self.id)) + finally: + self.mc_lock.release() + + def _load(self): + self.mc_lock.acquire() + try: + return self.cache.get(self.id) + finally: + self.mc_lock.release() + + def _save(self, expiration_time): + # Send the expiration time as "Unix time" (seconds since 1/1/1970) + td = int(time.mktime(expiration_time.timetuple())) + self.mc_lock.acquire() + try: + if not self.cache.set(self.id, (self._data, expiration_time), td): + raise AssertionError("Session data for id %r not set." % self.id) + finally: + self.mc_lock.release() + + def _delete(self): + self.cache.delete(self.id) + + def acquire_lock(self): + """Acquire an exclusive lock on the currently-loaded session data.""" + self.locked = True + self.locks.setdefault(self.id, threading.RLock()).acquire() + + def release_lock(self): + """Release the lock on the currently-loaded session data.""" + self.locks[self.id].release() + self.locked = False + + def __len__(self): + """Return the number of active sessions.""" + raise NotImplementedError + + +# Hook functions (for CherryPy tools) + +def save(): + """Save any changed session data.""" + + if not hasattr(cherrypy.serving, "session"): + return + request = cherrypy.serving.request + response = cherrypy.serving.response + + # Guard against running twice + if hasattr(request, "_sessionsaved"): + return + request._sessionsaved = True + + if response.stream: + # If the body is being streamed, we have to save the data + # *after* the response has been written out + request.hooks.attach('on_end_request', cherrypy.session.save) + else: + # If the body is not being streamed, we save the data now + # (so we can release the lock). + if isinstance(response.body, types.GeneratorType): + response.collapse_body() + cherrypy.session.save() +save.failsafe = True + +def close(): + """Close the session object for this request.""" + sess = getattr(cherrypy.serving, "session", None) + if getattr(sess, "locked", False): + # If the session is still locked we release the lock + sess.release_lock() +close.failsafe = True +close.priority = 90 + + +def init(storage_type='ram', path=None, path_header=None, name='session_id', + timeout=60, domain=None, secure=False, clean_freq=5, + persistent=True, debug=False, **kwargs): + """Initialize session object (using cookies). + + storage_type + One of 'ram', 'file', 'postgresql'. This will be used + to look up the corresponding class in cherrypy.lib.sessions + globals. For example, 'file' will use the FileSession class. + + path + The 'path' value to stick in the response cookie metadata. + + path_header + If 'path' is None (the default), then the response + cookie 'path' will be pulled from request.headers[path_header]. + + name + The name of the cookie. + + timeout + The expiration timeout (in minutes) for the stored session data. + If 'persistent' is True (the default), this is also the timeout + for the cookie. + + domain + The cookie domain. + + secure + If False (the default) the cookie 'secure' value will not + be set. If True, the cookie 'secure' value will be set (to 1). + + clean_freq (minutes) + The poll rate for expired session cleanup. + + persistent + If True (the default), the 'timeout' argument will be used + to expire the cookie. If False, the cookie will not have an expiry, + and the cookie will be a "session cookie" which expires when the + browser is closed. + + Any additional kwargs will be bound to the new Session instance, + and may be specific to the storage type. See the subclass of Session + you're using for more information. + """ + + request = cherrypy.serving.request + + # Guard against running twice + if hasattr(request, "_session_init_flag"): + return + request._session_init_flag = True + + # Check if request came with a session ID + id = None + if name in request.cookie: + id = request.cookie[name].value + if debug: + cherrypy.log('ID obtained from request.cookie: %r' % id, + 'TOOLS.SESSIONS') + + # Find the storage class and call setup (first time only). + storage_class = storage_type.title() + 'Session' + storage_class = globals()[storage_class] + if not hasattr(cherrypy, "session"): + if hasattr(storage_class, "setup"): + storage_class.setup(**kwargs) + + # Create and attach a new Session instance to cherrypy.serving. + # It will possess a reference to (and lock, and lazily load) + # the requested session data. + kwargs['timeout'] = timeout + kwargs['clean_freq'] = clean_freq + cherrypy.serving.session = sess = storage_class(id, **kwargs) + sess.debug = debug + def update_cookie(id): + """Update the cookie every time the session id changes.""" + cherrypy.serving.response.cookie[name] = id + sess.id_observers.append(update_cookie) + + # Create cherrypy.session which will proxy to cherrypy.serving.session + if not hasattr(cherrypy, "session"): + cherrypy.session = cherrypy._ThreadLocalProxy('session') + + if persistent: + cookie_timeout = timeout + else: + # See http://support.microsoft.com/kb/223799/EN-US/ + # and http://support.mozilla.com/en-US/kb/Cookies + cookie_timeout = None + set_response_cookie(path=path, path_header=path_header, name=name, + timeout=cookie_timeout, domain=domain, secure=secure) + + +def set_response_cookie(path=None, path_header=None, name='session_id', + timeout=60, domain=None, secure=False): + """Set a response cookie for the client. + + path + the 'path' value to stick in the response cookie metadata. + + path_header + if 'path' is None (the default), then the response + cookie 'path' will be pulled from request.headers[path_header]. + + name + the name of the cookie. + + timeout + the expiration timeout for the cookie. If 0 or other boolean + False, no 'expires' param will be set, and the cookie will be a + "session cookie" which expires when the browser is closed. + + domain + the cookie domain. + + secure + if False (the default) the cookie 'secure' value will not + be set. If True, the cookie 'secure' value will be set (to 1). + + """ + # Set response cookie + cookie = cherrypy.serving.response.cookie + cookie[name] = cherrypy.serving.session.id + cookie[name]['path'] = (path or cherrypy.serving.request.headers.get(path_header) + or '/') + + # We'd like to use the "max-age" param as indicated in + # http://www.faqs.org/rfcs/rfc2109.html but IE doesn't + # save it to disk and the session is lost if people close + # the browser. So we have to use the old "expires" ... sigh ... +## cookie[name]['max-age'] = timeout * 60 + if timeout: + e = time.time() + (timeout * 60) + cookie[name]['expires'] = httputil.HTTPDate(e) + if domain is not None: + cookie[name]['domain'] = domain + if secure: + cookie[name]['secure'] = 1 + + +def expire(): + """Expire the current session cookie.""" + name = cherrypy.serving.request.config.get('tools.sessions.name', 'session_id') + one_year = 60 * 60 * 24 * 365 + e = time.time() - one_year + cherrypy.serving.response.cookie[name]['expires'] = httputil.HTTPDate(e) + + diff --git a/cherrypy/lib/static.py b/cherrypy/lib/static.py new file mode 100755 index 0000000..cb9a68c --- /dev/null +++ b/cherrypy/lib/static.py @@ -0,0 +1,352 @@ +import logging +import mimetypes +mimetypes.init() +mimetypes.types_map['.dwg']='image/x-dwg' +mimetypes.types_map['.ico']='image/x-icon' +mimetypes.types_map['.bz2']='application/x-bzip2' +mimetypes.types_map['.gz']='application/x-gzip' + +import os +import re +import stat +import time + +import cherrypy +from cherrypy._cpcompat import ntob, unquote +from cherrypy.lib import cptools, httputil, file_generator_limited + + +def serve_file(path, content_type=None, disposition=None, name=None, debug=False): + """Set status, headers, and body in order to serve the given path. + + The Content-Type header will be set to the content_type arg, if provided. + If not provided, the Content-Type will be guessed by the file extension + of the 'path' argument. + + If disposition is not None, the Content-Disposition header will be set + to "<disposition>; filename=<name>". If name is None, it will be set + to the basename of path. If disposition is None, no Content-Disposition + header will be written. + """ + + response = cherrypy.serving.response + + # If path is relative, users should fix it by making path absolute. + # That is, CherryPy should not guess where the application root is. + # It certainly should *not* use cwd (since CP may be invoked from a + # variety of paths). If using tools.staticdir, you can make your relative + # paths become absolute by supplying a value for "tools.staticdir.root". + if not os.path.isabs(path): + msg = "'%s' is not an absolute path." % path + if debug: + cherrypy.log(msg, 'TOOLS.STATICFILE') + raise ValueError(msg) + + try: + st = os.stat(path) + except OSError: + if debug: + cherrypy.log('os.stat(%r) failed' % path, 'TOOLS.STATIC') + raise cherrypy.NotFound() + + # Check if path is a directory. + if stat.S_ISDIR(st.st_mode): + # Let the caller deal with it as they like. + if debug: + cherrypy.log('%r is a directory' % path, 'TOOLS.STATIC') + raise cherrypy.NotFound() + + # Set the Last-Modified response header, so that + # modified-since validation code can work. + response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime) + cptools.validate_since() + + if content_type is None: + # Set content-type based on filename extension + ext = "" + i = path.rfind('.') + if i != -1: + ext = path[i:].lower() + content_type = mimetypes.types_map.get(ext, None) + if content_type is not None: + response.headers['Content-Type'] = content_type + if debug: + cherrypy.log('Content-Type: %r' % content_type, 'TOOLS.STATIC') + + cd = None + if disposition is not None: + if name is None: + name = os.path.basename(path) + cd = '%s; filename="%s"' % (disposition, name) + response.headers["Content-Disposition"] = cd + if debug: + cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') + + # Set Content-Length and use an iterable (file object) + # this way CP won't load the whole file in memory + content_length = st.st_size + fileobj = open(path, 'rb') + return _serve_fileobj(fileobj, content_type, content_length, debug=debug) + +def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, + debug=False): + """Set status, headers, and body in order to serve the given file object. + + The Content-Type header will be set to the content_type arg, if provided. + + If disposition is not None, the Content-Disposition header will be set + to "<disposition>; filename=<name>". If name is None, 'filename' will + not be set. If disposition is None, no Content-Disposition header will + be written. + + CAUTION: If the request contains a 'Range' header, one or more seek()s will + be performed on the file object. This may cause undesired behavior if + the file object is not seekable. It could also produce undesired results + if the caller set the read position of the file object prior to calling + serve_fileobj(), expecting that the data would be served starting from that + position. + """ + + response = cherrypy.serving.response + + try: + st = os.fstat(fileobj.fileno()) + except AttributeError: + if debug: + cherrypy.log('os has no fstat attribute', 'TOOLS.STATIC') + content_length = None + else: + # Set the Last-Modified response header, so that + # modified-since validation code can work. + response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime) + cptools.validate_since() + content_length = st.st_size + + if content_type is not None: + response.headers['Content-Type'] = content_type + if debug: + cherrypy.log('Content-Type: %r' % content_type, 'TOOLS.STATIC') + + cd = None + if disposition is not None: + if name is None: + cd = disposition + else: + cd = '%s; filename="%s"' % (disposition, name) + response.headers["Content-Disposition"] = cd + if debug: + cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') + + return _serve_fileobj(fileobj, content_type, content_length, debug=debug) + +def _serve_fileobj(fileobj, content_type, content_length, debug=False): + """Internal. Set response.body to the given file object, perhaps ranged.""" + response = cherrypy.serving.response + + # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code + request = cherrypy.serving.request + if request.protocol >= (1, 1): + response.headers["Accept-Ranges"] = "bytes" + r = httputil.get_ranges(request.headers.get('Range'), content_length) + if r == []: + response.headers['Content-Range'] = "bytes */%s" % content_length + message = "Invalid Range (first-byte-pos greater than Content-Length)" + if debug: + cherrypy.log(message, 'TOOLS.STATIC') + raise cherrypy.HTTPError(416, message) + + if r: + if len(r) == 1: + # Return a single-part response. + start, stop = r[0] + if stop > content_length: + stop = content_length + r_len = stop - start + if debug: + cherrypy.log('Single part; start: %r, stop: %r' % (start, stop), + 'TOOLS.STATIC') + response.status = "206 Partial Content" + response.headers['Content-Range'] = ( + "bytes %s-%s/%s" % (start, stop - 1, content_length)) + response.headers['Content-Length'] = r_len + fileobj.seek(start) + response.body = file_generator_limited(fileobj, r_len) + else: + # Return a multipart/byteranges response. + response.status = "206 Partial Content" + from mimetools import choose_boundary + boundary = choose_boundary() + ct = "multipart/byteranges; boundary=%s" % boundary + response.headers['Content-Type'] = ct + if "Content-Length" in response.headers: + # Delete Content-Length header so finalize() recalcs it. + del response.headers["Content-Length"] + + def file_ranges(): + # Apache compatibility: + yield ntob("\r\n") + + for start, stop in r: + if debug: + cherrypy.log('Multipart; start: %r, stop: %r' % (start, stop), + 'TOOLS.STATIC') + yield ntob("--" + boundary, 'ascii') + yield ntob("\r\nContent-type: %s" % content_type, 'ascii') + yield ntob("\r\nContent-range: bytes %s-%s/%s\r\n\r\n" + % (start, stop - 1, content_length), 'ascii') + fileobj.seek(start) + for chunk in file_generator_limited(fileobj, stop-start): + yield chunk + yield ntob("\r\n") + # Final boundary + yield ntob("--" + boundary + "--", 'ascii') + + # Apache compatibility: + yield ntob("\r\n") + response.body = file_ranges() + return response.body + else: + if debug: + cherrypy.log('No byteranges requested', 'TOOLS.STATIC') + + # Set Content-Length and use an iterable (file object) + # this way CP won't load the whole file in memory + response.headers['Content-Length'] = content_length + response.body = fileobj + return response.body + +def serve_download(path, name=None): + """Serve 'path' as an application/x-download attachment.""" + # This is such a common idiom I felt it deserved its own wrapper. + return serve_file(path, "application/x-download", "attachment", name) + + +def _attempt(filename, content_types, debug=False): + if debug: + cherrypy.log('Attempting %r (content_types %r)' % + (filename, content_types), 'TOOLS.STATICDIR') + try: + # you can set the content types for a + # complete directory per extension + content_type = None + if content_types: + r, ext = os.path.splitext(filename) + content_type = content_types.get(ext[1:], None) + serve_file(filename, content_type=content_type, debug=debug) + return True + except cherrypy.NotFound: + # If we didn't find the static file, continue handling the + # request. We might find a dynamic handler instead. + if debug: + cherrypy.log('NotFound', 'TOOLS.STATICFILE') + return False + +def staticdir(section, dir, root="", match="", content_types=None, index="", + debug=False): + """Serve a static resource from the given (root +) dir. + + match + If given, request.path_info will be searched for the given + regular expression before attempting to serve static content. + + content_types + If given, it should be a Python dictionary of + {file-extension: content-type} pairs, where 'file-extension' is + a string (e.g. "gif") and 'content-type' is the value to write + out in the Content-Type response header (e.g. "image/gif"). + + index + If provided, it should be the (relative) name of a file to + serve for directory requests. For example, if the dir argument is + '/home/me', the Request-URI is 'myapp', and the index arg is + 'index.html', the file '/home/me/myapp/index.html' will be sought. + """ + request = cherrypy.serving.request + if request.method not in ('GET', 'HEAD'): + if debug: + cherrypy.log('request.method not GET or HEAD', 'TOOLS.STATICDIR') + return False + + if match and not re.search(match, request.path_info): + if debug: + cherrypy.log('request.path_info %r does not match pattern %r' % + (request.path_info, match), 'TOOLS.STATICDIR') + return False + + # Allow the use of '~' to refer to a user's home directory. + dir = os.path.expanduser(dir) + + # If dir is relative, make absolute using "root". + if not os.path.isabs(dir): + if not root: + msg = "Static dir requires an absolute dir (or root)." + if debug: + cherrypy.log(msg, 'TOOLS.STATICDIR') + raise ValueError(msg) + dir = os.path.join(root, dir) + + # Determine where we are in the object tree relative to 'section' + # (where the static tool was defined). + if section == 'global': + section = "/" + section = section.rstrip(r"\/") + branch = request.path_info[len(section) + 1:] + branch = unquote(branch.lstrip(r"\/")) + + # If branch is "", filename will end in a slash + filename = os.path.join(dir, branch) + if debug: + cherrypy.log('Checking file %r to fulfill %r' % + (filename, request.path_info), 'TOOLS.STATICDIR') + + # There's a chance that the branch pulled from the URL might + # have ".." or similar uplevel attacks in it. Check that the final + # filename is a child of dir. + if not os.path.normpath(filename).startswith(os.path.normpath(dir)): + raise cherrypy.HTTPError(403) # Forbidden + + handled = _attempt(filename, content_types) + if not handled: + # Check for an index file if a folder was requested. + if index: + handled = _attempt(os.path.join(filename, index), content_types) + if handled: + request.is_index = filename[-1] in (r"\/") + return handled + +def staticfile(filename, root=None, match="", content_types=None, debug=False): + """Serve a static resource from the given (root +) filename. + + match + If given, request.path_info will be searched for the given + regular expression before attempting to serve static content. + + content_types + If given, it should be a Python dictionary of + {file-extension: content-type} pairs, where 'file-extension' is + a string (e.g. "gif") and 'content-type' is the value to write + out in the Content-Type response header (e.g. "image/gif"). + + """ + request = cherrypy.serving.request + if request.method not in ('GET', 'HEAD'): + if debug: + cherrypy.log('request.method not GET or HEAD', 'TOOLS.STATICFILE') + return False + + if match and not re.search(match, request.path_info): + if debug: + cherrypy.log('request.path_info %r does not match pattern %r' % + (request.path_info, match), 'TOOLS.STATICFILE') + return False + + # If filename is relative, make absolute using "root". + if not os.path.isabs(filename): + if not root: + msg = "Static tool requires an absolute filename (got '%s')." % filename + if debug: + cherrypy.log(msg, 'TOOLS.STATICFILE') + raise ValueError(msg) + filename = os.path.join(root, filename) + + return _attempt(filename, content_types, debug=debug) diff --git a/cherrypy/lib/xmlrpc.py b/cherrypy/lib/xmlrpc.py new file mode 100755 index 0000000..8a5ef54 --- /dev/null +++ b/cherrypy/lib/xmlrpc.py @@ -0,0 +1,49 @@ +import sys + +import cherrypy + + +def process_body(): + """Return (params, method) from request body.""" + try: + import xmlrpclib + return xmlrpclib.loads(cherrypy.request.body.read()) + except Exception: + return ('ERROR PARAMS', ), 'ERRORMETHOD' + + +def patched_path(path): + """Return 'path', doctored for RPC.""" + if not path.endswith('/'): + path += '/' + if path.startswith('/RPC2/'): + # strip the first /rpc2 + path = path[5:] + return path + + +def _set_response(body): + # The XML-RPC spec (http://www.xmlrpc.com/spec) says: + # "Unless there's a lower-level error, always return 200 OK." + # Since Python's xmlrpclib interprets a non-200 response + # as a "Protocol Error", we'll just return 200 every time. + response = cherrypy.response + response.status = '200 OK' + response.body = body + response.headers['Content-Type'] = 'text/xml' + response.headers['Content-Length'] = len(body) + + +def respond(body, encoding='utf-8', allow_none=0): + from xmlrpclib import Fault, dumps + if not isinstance(body, Fault): + body = (body,) + _set_response(dumps(body, methodresponse=1, + encoding=encoding, + allow_none=allow_none)) + +def on_error(*args, **kwargs): + body = str(sys.exc_info()[1]) + from xmlrpclib import Fault, dumps + _set_response(dumps(Fault(1, body))) + diff --git a/cherrypy/process/__init__.py b/cherrypy/process/__init__.py new file mode 100755 index 0000000..f15b123 --- /dev/null +++ b/cherrypy/process/__init__.py @@ -0,0 +1,14 @@ +"""Site container for an HTTP server. + +A Web Site Process Bus object is used to connect applications, servers, +and frameworks with site-wide services such as daemonization, process +reload, signal handling, drop privileges, PID file management, logging +for all of these, and many more. + +The 'plugins' module defines a few abstract and concrete services for +use with the bus. Some use tool-specific channels; see the documentation +for each class. +""" + +from cherrypy.process.wspbus import bus +from cherrypy.process import plugins, servers diff --git a/cherrypy/process/plugins.py b/cherrypy/process/plugins.py new file mode 100755 index 0000000..488958e --- /dev/null +++ b/cherrypy/process/plugins.py @@ -0,0 +1,681 @@ +"""Site services for use with a Web Site Process Bus.""" + +import os +import re +import signal as _signal +import sys +import time +import threading + +from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident, ntob, set + +# _module__file__base is used by Autoreload to make +# absolute any filenames retrieved from sys.modules which are not +# already absolute paths. This is to work around Python's quirk +# of importing the startup script and using a relative filename +# for it in sys.modules. +# +# Autoreload examines sys.modules afresh every time it runs. If an application +# changes the current directory by executing os.chdir(), then the next time +# Autoreload runs, it will not be able to find any filenames which are +# not absolute paths, because the current directory is not the same as when the +# module was first imported. Autoreload will then wrongly conclude the file has +# "changed", and initiate the shutdown/re-exec sequence. +# See ticket #917. +# For this workaround to have a decent probability of success, this module +# needs to be imported as early as possible, before the app has much chance +# to change the working directory. +_module__file__base = os.getcwd() + + +class SimplePlugin(object): + """Plugin base class which auto-subscribes methods for known channels.""" + + bus = None + """A :class:`Bus <cherrypy.process.wspbus.Bus>`, usually cherrypy.engine.""" + + def __init__(self, bus): + self.bus = bus + + def subscribe(self): + """Register this object as a (multi-channel) listener on the bus.""" + for channel in self.bus.listeners: + # Subscribe self.start, self.exit, etc. if present. + method = getattr(self, channel, None) + if method is not None: + self.bus.subscribe(channel, method) + + def unsubscribe(self): + """Unregister this object as a listener on the bus.""" + for channel in self.bus.listeners: + # Unsubscribe self.start, self.exit, etc. if present. + method = getattr(self, channel, None) + if method is not None: + self.bus.unsubscribe(channel, method) + + + +class SignalHandler(object): + """Register bus channels (and listeners) for system signals. + + You can modify what signals your application listens for, and what it does + when it receives signals, by modifying :attr:`SignalHandler.handlers`, + a dict of {signal name: callback} pairs. The default set is:: + + handlers = {'SIGTERM': self.bus.exit, + 'SIGHUP': self.handle_SIGHUP, + 'SIGUSR1': self.bus.graceful, + } + + The :func:`SignalHandler.handle_SIGHUP`` method calls + :func:`bus.restart()<cherrypy.process.wspbus.Bus.restart>` + if the process is daemonized, but + :func:`bus.exit()<cherrypy.process.wspbus.Bus.exit>` + if the process is attached to a TTY. This is because Unix window + managers tend to send SIGHUP to terminal windows when the user closes them. + + Feel free to add signals which are not available on every platform. The + :class:`SignalHandler` will ignore errors raised from attempting to register + handlers for unknown signals. + """ + + handlers = {} + """A map from signal names (e.g. 'SIGTERM') to handlers (e.g. bus.exit).""" + + signals = {} + """A map from signal numbers to names.""" + + for k, v in vars(_signal).items(): + if k.startswith('SIG') and not k.startswith('SIG_'): + signals[v] = k + del k, v + + def __init__(self, bus): + self.bus = bus + # Set default handlers + self.handlers = {'SIGTERM': self.bus.exit, + 'SIGHUP': self.handle_SIGHUP, + 'SIGUSR1': self.bus.graceful, + } + + if sys.platform[:4] == 'java': + del self.handlers['SIGUSR1'] + self.handlers['SIGUSR2'] = self.bus.graceful + self.bus.log("SIGUSR1 cannot be set on the JVM platform. " + "Using SIGUSR2 instead.") + self.handlers['SIGINT'] = self._jython_SIGINT_handler + + self._previous_handlers = {} + + def _jython_SIGINT_handler(self, signum=None, frame=None): + # See http://bugs.jython.org/issue1313 + self.bus.log('Keyboard Interrupt: shutting down bus') + self.bus.exit() + + def subscribe(self): + """Subscribe self.handlers to signals.""" + for sig, func in self.handlers.items(): + try: + self.set_handler(sig, func) + except ValueError: + pass + + def unsubscribe(self): + """Unsubscribe self.handlers from signals.""" + for signum, handler in self._previous_handlers.items(): + signame = self.signals[signum] + + if handler is None: + self.bus.log("Restoring %s handler to SIG_DFL." % signame) + handler = _signal.SIG_DFL + else: + self.bus.log("Restoring %s handler %r." % (signame, handler)) + + try: + our_handler = _signal.signal(signum, handler) + if our_handler is None: + self.bus.log("Restored old %s handler %r, but our " + "handler was not registered." % + (signame, handler), level=30) + except ValueError: + self.bus.log("Unable to restore %s handler %r." % + (signame, handler), level=40, traceback=True) + + def set_handler(self, signal, listener=None): + """Subscribe a handler for the given signal (number or name). + + If the optional 'listener' argument is provided, it will be + subscribed as a listener for the given signal's channel. + + If the given signal name or number is not available on the current + platform, ValueError is raised. + """ + if isinstance(signal, basestring): + signum = getattr(_signal, signal, None) + if signum is None: + raise ValueError("No such signal: %r" % signal) + signame = signal + else: + try: + signame = self.signals[signal] + except KeyError: + raise ValueError("No such signal: %r" % signal) + signum = signal + + prev = _signal.signal(signum, self._handle_signal) + self._previous_handlers[signum] = prev + + if listener is not None: + self.bus.log("Listening for %s." % signame) + self.bus.subscribe(signame, listener) + + def _handle_signal(self, signum=None, frame=None): + """Python signal handler (self.set_handler subscribes it for you).""" + signame = self.signals[signum] + self.bus.log("Caught signal %s." % signame) + self.bus.publish(signame) + + def handle_SIGHUP(self): + """Restart if daemonized, else exit.""" + if os.isatty(sys.stdin.fileno()): + # not daemonized (may be foreground or background) + self.bus.log("SIGHUP caught but not daemonized. Exiting.") + self.bus.exit() + else: + self.bus.log("SIGHUP caught while daemonized. Restarting.") + self.bus.restart() + + +try: + import pwd, grp +except ImportError: + pwd, grp = None, None + + +class DropPrivileges(SimplePlugin): + """Drop privileges. uid/gid arguments not available on Windows. + + Special thanks to Gavin Baker: http://antonym.org/node/100. + """ + + def __init__(self, bus, umask=None, uid=None, gid=None): + SimplePlugin.__init__(self, bus) + self.finalized = False + self.uid = uid + self.gid = gid + self.umask = umask + + def _get_uid(self): + return self._uid + def _set_uid(self, val): + if val is not None: + if pwd is None: + self.bus.log("pwd module not available; ignoring uid.", + level=30) + val = None + elif isinstance(val, basestring): + val = pwd.getpwnam(val)[2] + self._uid = val + uid = property(_get_uid, _set_uid, + doc="The uid under which to run. Availability: Unix.") + + def _get_gid(self): + return self._gid + def _set_gid(self, val): + if val is not None: + if grp is None: + self.bus.log("grp module not available; ignoring gid.", + level=30) + val = None + elif isinstance(val, basestring): + val = grp.getgrnam(val)[2] + self._gid = val + gid = property(_get_gid, _set_gid, + doc="The gid under which to run. Availability: Unix.") + + def _get_umask(self): + return self._umask + def _set_umask(self, val): + if val is not None: + try: + os.umask + except AttributeError: + self.bus.log("umask function not available; ignoring umask.", + level=30) + val = None + self._umask = val + umask = property(_get_umask, _set_umask, + doc="""The default permission mode for newly created files and directories. + + Usually expressed in octal format, for example, ``0644``. + Availability: Unix, Windows. + """) + + def start(self): + # uid/gid + def current_ids(): + """Return the current (uid, gid) if available.""" + name, group = None, None + if pwd: + name = pwd.getpwuid(os.getuid())[0] + if grp: + group = grp.getgrgid(os.getgid())[0] + return name, group + + if self.finalized: + if not (self.uid is None and self.gid is None): + self.bus.log('Already running as uid: %r gid: %r' % + current_ids()) + else: + if self.uid is None and self.gid is None: + if pwd or grp: + self.bus.log('uid/gid not set', level=30) + else: + self.bus.log('Started as uid: %r gid: %r' % current_ids()) + if self.gid is not None: + os.setgid(self.gid) + os.setgroups([]) + if self.uid is not None: + os.setuid(self.uid) + self.bus.log('Running as uid: %r gid: %r' % current_ids()) + + # umask + if self.finalized: + if self.umask is not None: + self.bus.log('umask already set to: %03o' % self.umask) + else: + if self.umask is None: + self.bus.log('umask not set', level=30) + else: + old_umask = os.umask(self.umask) + self.bus.log('umask old: %03o, new: %03o' % + (old_umask, self.umask)) + + self.finalized = True + # This is slightly higher than the priority for server.start + # in order to facilitate the most common use: starting on a low + # port (which requires root) and then dropping to another user. + start.priority = 77 + + +class Daemonizer(SimplePlugin): + """Daemonize the running script. + + Use this with a Web Site Process Bus via:: + + Daemonizer(bus).subscribe() + + When this component finishes, the process is completely decoupled from + the parent environment. Please note that when this component is used, + the return code from the parent process will still be 0 if a startup + error occurs in the forked children. Errors in the initial daemonizing + process still return proper exit codes. Therefore, if you use this + plugin to daemonize, don't use the return code as an accurate indicator + of whether the process fully started. In fact, that return code only + indicates if the process succesfully finished the first fork. + """ + + def __init__(self, bus, stdin='/dev/null', stdout='/dev/null', + stderr='/dev/null'): + SimplePlugin.__init__(self, bus) + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self.finalized = False + + def start(self): + if self.finalized: + self.bus.log('Already deamonized.') + + # forking has issues with threads: + # http://www.opengroup.org/onlinepubs/000095399/functions/fork.html + # "The general problem with making fork() work in a multi-threaded + # world is what to do with all of the threads..." + # So we check for active threads: + if threading.activeCount() != 1: + self.bus.log('There are %r active threads. ' + 'Daemonizing now may cause strange failures.' % + threading.enumerate(), level=30) + + # See http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 + # (or http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7) + # and http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012 + + # Finish up with the current stdout/stderr + sys.stdout.flush() + sys.stderr.flush() + + # Do first fork. + try: + pid = os.fork() + if pid == 0: + # This is the child process. Continue. + pass + else: + # This is the first parent. Exit, now that we've forked. + self.bus.log('Forking once.') + os._exit(0) + except OSError: + # Python raises OSError rather than returning negative numbers. + exc = sys.exc_info()[1] + sys.exit("%s: fork #1 failed: (%d) %s\n" + % (sys.argv[0], exc.errno, exc.strerror)) + + os.setsid() + + # Do second fork + try: + pid = os.fork() + if pid > 0: + self.bus.log('Forking twice.') + os._exit(0) # Exit second parent + except OSError: + exc = sys.exc_info()[1] + sys.exit("%s: fork #2 failed: (%d) %s\n" + % (sys.argv[0], exc.errno, exc.strerror)) + + os.chdir("/") + os.umask(0) + + si = open(self.stdin, "r") + so = open(self.stdout, "a+") + se = open(self.stderr, "a+") + + # os.dup2(fd, fd2) will close fd2 if necessary, + # so we don't explicitly close stdin/out/err. + # See http://docs.python.org/lib/os-fd-ops.html + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + self.bus.log('Daemonized to PID: %s' % os.getpid()) + self.finalized = True + start.priority = 65 + + +class PIDFile(SimplePlugin): + """Maintain a PID file via a WSPBus.""" + + def __init__(self, bus, pidfile): + SimplePlugin.__init__(self, bus) + self.pidfile = pidfile + self.finalized = False + + def start(self): + pid = os.getpid() + if self.finalized: + self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) + else: + open(self.pidfile, "wb").write(ntob("%s" % pid, 'utf8')) + self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) + self.finalized = True + start.priority = 70 + + def exit(self): + try: + os.remove(self.pidfile) + self.bus.log('PID file removed: %r.' % self.pidfile) + except (KeyboardInterrupt, SystemExit): + raise + except: + pass + + +class PerpetualTimer(threading._Timer): + """A responsive subclass of threading._Timer whose run() method repeats. + + Use this timer only when you really need a very interruptible timer; + this checks its 'finished' condition up to 20 times a second, which can + results in pretty high CPU usage + """ + + def run(self): + while True: + self.finished.wait(self.interval) + if self.finished.isSet(): + return + try: + self.function(*self.args, **self.kwargs) + except Exception: + self.bus.log("Error in perpetual timer thread function %r." % + self.function, level=40, traceback=True) + # Quit on first error to avoid massive logs. + raise + + +class BackgroundTask(threading.Thread): + """A subclass of threading.Thread whose run() method repeats. + + Use this class for most repeating tasks. It uses time.sleep() to wait + for each interval, which isn't very responsive; that is, even if you call + self.cancel(), you'll have to wait until the sleep() call finishes before + the thread stops. To compensate, it defaults to being daemonic, which means + it won't delay stopping the whole process. + """ + + def __init__(self, interval, function, args=[], kwargs={}): + threading.Thread.__init__(self) + self.interval = interval + self.function = function + self.args = args + self.kwargs = kwargs + self.running = False + + def cancel(self): + self.running = False + + def run(self): + self.running = True + while self.running: + time.sleep(self.interval) + if not self.running: + return + try: + self.function(*self.args, **self.kwargs) + except Exception: + self.bus.log("Error in background task thread function %r." % + self.function, level=40, traceback=True) + # Quit on first error to avoid massive logs. + raise + + def _set_daemon(self): + return True + + +class Monitor(SimplePlugin): + """WSPBus listener to periodically run a callback in its own thread.""" + + callback = None + """The function to call at intervals.""" + + frequency = 60 + """The time in seconds between callback runs.""" + + thread = None + """A :class:`BackgroundTask<cherrypy.process.plugins.BackgroundTask>` thread.""" + + def __init__(self, bus, callback, frequency=60, name=None): + SimplePlugin.__init__(self, bus) + self.callback = callback + self.frequency = frequency + self.thread = None + self.name = name + + def start(self): + """Start our callback in its own background thread.""" + if self.frequency > 0: + threadname = self.name or self.__class__.__name__ + if self.thread is None: + self.thread = BackgroundTask(self.frequency, self.callback) + self.thread.bus = self.bus + self.thread.setName(threadname) + self.thread.start() + self.bus.log("Started monitor thread %r." % threadname) + else: + self.bus.log("Monitor thread %r already started." % threadname) + start.priority = 70 + + def stop(self): + """Stop our callback's background task thread.""" + if self.thread is None: + self.bus.log("No thread running for %s." % self.name or self.__class__.__name__) + else: + if self.thread is not threading.currentThread(): + name = self.thread.getName() + self.thread.cancel() + if not get_daemon(self.thread): + self.bus.log("Joining %r" % name) + self.thread.join() + self.bus.log("Stopped thread %r." % name) + self.thread = None + + def graceful(self): + """Stop the callback's background task thread and restart it.""" + self.stop() + self.start() + + +class Autoreloader(Monitor): + """Monitor which re-executes the process when files change. + + This :ref:`plugin<plugins>` restarts the process (via :func:`os.execv`) + if any of the files it monitors change (or is deleted). By default, the + autoreloader monitors all imported modules; you can add to the + set by adding to ``autoreload.files``:: + + cherrypy.engine.autoreload.files.add(myFile) + + If there are imported files you do *not* wish to monitor, you can adjust the + ``match`` attribute, a regular expression. For example, to stop monitoring + cherrypy itself:: + + cherrypy.engine.autoreload.match = r'^(?!cherrypy).+' + + Like all :class:`Monitor<cherrypy.process.plugins.Monitor>` plugins, + the autoreload plugin takes a ``frequency`` argument. The default is + 1 second; that is, the autoreloader will examine files once each second. + """ + + files = None + """The set of files to poll for modifications.""" + + frequency = 1 + """The interval in seconds at which to poll for modified files.""" + + match = '.*' + """A regular expression by which to match filenames.""" + + def __init__(self, bus, frequency=1, match='.*'): + self.mtimes = {} + self.files = set() + self.match = match + Monitor.__init__(self, bus, self.run, frequency) + + def start(self): + """Start our own background task thread for self.run.""" + if self.thread is None: + self.mtimes = {} + Monitor.start(self) + start.priority = 70 + + def sysfiles(self): + """Return a Set of sys.modules filenames to monitor.""" + files = set() + for k, m in sys.modules.items(): + if re.match(self.match, k): + if hasattr(m, '__loader__') and hasattr(m.__loader__, 'archive'): + f = m.__loader__.archive + else: + f = getattr(m, '__file__', None) + if f is not None and not os.path.isabs(f): + # ensure absolute paths so a os.chdir() in the app doesn't break me + f = os.path.normpath(os.path.join(_module__file__base, f)) + files.add(f) + return files + + def run(self): + """Reload the process if registered files have been modified.""" + for filename in self.sysfiles() | self.files: + if filename: + if filename.endswith('.pyc'): + filename = filename[:-1] + + oldtime = self.mtimes.get(filename, 0) + if oldtime is None: + # Module with no .py file. Skip it. + continue + + try: + mtime = os.stat(filename).st_mtime + except OSError: + # Either a module with no .py file, or it's been deleted. + mtime = None + + if filename not in self.mtimes: + # If a module has no .py file, this will be None. + self.mtimes[filename] = mtime + else: + if mtime is None or mtime > oldtime: + # The file has been deleted or modified. + self.bus.log("Restarting because %s changed." % filename) + self.thread.cancel() + self.bus.log("Stopped thread %r." % self.thread.getName()) + self.bus.restart() + return + + +class ThreadManager(SimplePlugin): + """Manager for HTTP request threads. + + If you have control over thread creation and destruction, publish to + the 'acquire_thread' and 'release_thread' channels (for each thread). + This will register/unregister the current thread and publish to + 'start_thread' and 'stop_thread' listeners in the bus as needed. + + If threads are created and destroyed by code you do not control + (e.g., Apache), then, at the beginning of every HTTP request, + publish to 'acquire_thread' only. You should not publish to + 'release_thread' in this case, since you do not know whether + the thread will be re-used or not. The bus will call + 'stop_thread' listeners for you when it stops. + """ + + threads = None + """A map of {thread ident: index number} pairs.""" + + def __init__(self, bus): + self.threads = {} + SimplePlugin.__init__(self, bus) + self.bus.listeners.setdefault('acquire_thread', set()) + self.bus.listeners.setdefault('start_thread', set()) + self.bus.listeners.setdefault('release_thread', set()) + self.bus.listeners.setdefault('stop_thread', set()) + + def acquire_thread(self): + """Run 'start_thread' listeners for the current thread. + + If the current thread has already been seen, any 'start_thread' + listeners will not be run again. + """ + thread_ident = get_thread_ident() + if thread_ident not in self.threads: + # We can't just use get_ident as the thread ID + # because some platforms reuse thread ID's. + i = len(self.threads) + 1 + self.threads[thread_ident] = i + self.bus.publish('start_thread', i) + + def release_thread(self): + """Release the current thread and run 'stop_thread' listeners.""" + thread_ident = get_thread_ident() + i = self.threads.pop(thread_ident, None) + if i is not None: + self.bus.publish('stop_thread', i) + + def stop(self): + """Release all threads and run all 'stop_thread' listeners.""" + for thread_ident, i in self.threads.items(): + self.bus.publish('stop_thread', i) + self.threads.clear() + graceful = stop + diff --git a/cherrypy/process/servers.py b/cherrypy/process/servers.py new file mode 100755 index 0000000..272e843 --- /dev/null +++ b/cherrypy/process/servers.py @@ -0,0 +1,418 @@ +""" +Starting in CherryPy 3.1, cherrypy.server is implemented as an +:ref:`Engine Plugin<plugins>`. It's an instance of +:class:`cherrypy._cpserver.Server`, which is a subclass of +:class:`cherrypy.process.servers.ServerAdapter`. The ``ServerAdapter`` class +is designed to control other servers, as well. + +Multiple servers/ports +====================== + +If you need to start more than one HTTP server (to serve on multiple ports, or +protocols, etc.), you can manually register each one and then start them all +with engine.start:: + + s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80)) + s2 = ServerAdapter(cherrypy.engine, another.HTTPServer(host='127.0.0.1', SSL=True)) + s1.subscribe() + s2.subscribe() + cherrypy.engine.start() + +.. index:: SCGI + +FastCGI/SCGI +============ + +There are also Flup\ **F**\ CGIServer and Flup\ **S**\ CGIServer classes in +:mod:`cherrypy.process.servers`. To start an fcgi server, for example, +wrap an instance of it in a ServerAdapter:: + + addr = ('0.0.0.0', 4000) + f = servers.FlupFCGIServer(application=cherrypy.tree, bindAddress=addr) + s = servers.ServerAdapter(cherrypy.engine, httpserver=f, bind_addr=addr) + s.subscribe() + +The :doc:`cherryd</deployguide/cherryd>` startup script will do the above for +you via its `-f` flag. +Note that you need to download and install `flup <http://trac.saddi.com/flup>`_ +yourself, whether you use ``cherryd`` or not. + +.. _fastcgi: +.. index:: FastCGI + +FastCGI +------- + +A very simple setup lets your cherry run with FastCGI. +You just need the flup library, +plus a running Apache server (with ``mod_fastcgi``) or lighttpd server. + +CherryPy code +^^^^^^^^^^^^^ + +hello.py:: + + #!/usr/bin/python + import cherrypy + + class HelloWorld: + \"""Sample request handler class.\""" + def index(self): + return "Hello world!" + index.exposed = True + + cherrypy.tree.mount(HelloWorld()) + # CherryPy autoreload must be disabled for the flup server to work + cherrypy.config.update({'engine.autoreload_on':False}) + +Then run :doc:`/deployguide/cherryd` with the '-f' arg:: + + cherryd -c <myconfig> -d -f -i hello.py + +Apache +^^^^^^ + +At the top level in httpd.conf:: + + FastCgiIpcDir /tmp + FastCgiServer /path/to/cherry.fcgi -idle-timeout 120 -processes 4 + +And inside the relevant VirtualHost section:: + + # FastCGI config + AddHandler fastcgi-script .fcgi + ScriptAliasMatch (.*$) /path/to/cherry.fcgi$1 + +Lighttpd +^^^^^^^^ + +For `Lighttpd <http://www.lighttpd.net/>`_ you can follow these +instructions. Within ``lighttpd.conf`` make sure ``mod_fastcgi`` is +active within ``server.modules``. Then, within your ``$HTTP["host"]`` +directive, configure your fastcgi script like the following:: + + $HTTP["url"] =~ "" { + fastcgi.server = ( + "/" => ( + "script.fcgi" => ( + "bin-path" => "/path/to/your/script.fcgi", + "socket" => "/tmp/script.sock", + "check-local" => "disable", + "disable-time" => 1, + "min-procs" => 1, + "max-procs" => 1, # adjust as needed + ), + ), + ) + } # end of $HTTP["url"] =~ "^/" + +Please see `Lighttpd FastCGI Docs +<http://redmine.lighttpd.net/wiki/lighttpd/Docs:ModFastCGI>`_ for an explanation +of the possible configuration options. +""" + +import sys +import time + + +class ServerAdapter(object): + """Adapter for an HTTP server. + + If you need to start more than one HTTP server (to serve on multiple + ports, or protocols, etc.), you can manually register each one and then + start them all with bus.start: + + s1 = ServerAdapter(bus, MyWSGIServer(host='0.0.0.0', port=80)) + s2 = ServerAdapter(bus, another.HTTPServer(host='127.0.0.1', SSL=True)) + s1.subscribe() + s2.subscribe() + bus.start() + """ + + def __init__(self, bus, httpserver=None, bind_addr=None): + self.bus = bus + self.httpserver = httpserver + self.bind_addr = bind_addr + self.interrupt = None + self.running = False + + def subscribe(self): + self.bus.subscribe('start', self.start) + self.bus.subscribe('stop', self.stop) + + def unsubscribe(self): + self.bus.unsubscribe('start', self.start) + self.bus.unsubscribe('stop', self.stop) + + def start(self): + """Start the HTTP server.""" + if self.bind_addr is None: + on_what = "unknown interface (dynamic?)" + elif isinstance(self.bind_addr, tuple): + host, port = self.bind_addr + on_what = "%s:%s" % (host, port) + else: + on_what = "socket file: %s" % self.bind_addr + + if self.running: + self.bus.log("Already serving on %s" % on_what) + return + + self.interrupt = None + if not self.httpserver: + raise ValueError("No HTTP server has been created.") + + # Start the httpserver in a new thread. + if isinstance(self.bind_addr, tuple): + wait_for_free_port(*self.bind_addr) + + import threading + t = threading.Thread(target=self._start_http_thread) + t.setName("HTTPServer " + t.getName()) + t.start() + + self.wait() + self.running = True + self.bus.log("Serving on %s" % on_what) + start.priority = 75 + + def _start_http_thread(self): + """HTTP servers MUST be running in new threads, so that the + main thread persists to receive KeyboardInterrupt's. If an + exception is raised in the httpserver's thread then it's + trapped here, and the bus (and therefore our httpserver) + are shut down. + """ + try: + self.httpserver.start() + except KeyboardInterrupt: + self.bus.log("<Ctrl-C> hit: shutting down HTTP server") + self.interrupt = sys.exc_info()[1] + self.bus.exit() + except SystemExit: + self.bus.log("SystemExit raised: shutting down HTTP server") + self.interrupt = sys.exc_info()[1] + self.bus.exit() + raise + except: + self.interrupt = sys.exc_info()[1] + self.bus.log("Error in HTTP server: shutting down", + traceback=True, level=40) + self.bus.exit() + raise + + def wait(self): + """Wait until the HTTP server is ready to receive requests.""" + while not getattr(self.httpserver, "ready", False): + if self.interrupt: + raise self.interrupt + time.sleep(.1) + + # Wait for port to be occupied + if isinstance(self.bind_addr, tuple): + host, port = self.bind_addr + wait_for_occupied_port(host, port) + + def stop(self): + """Stop the HTTP server.""" + if self.running: + # stop() MUST block until the server is *truly* stopped. + self.httpserver.stop() + # Wait for the socket to be truly freed. + if isinstance(self.bind_addr, tuple): + wait_for_free_port(*self.bind_addr) + self.running = False + self.bus.log("HTTP Server %s shut down" % self.httpserver) + else: + self.bus.log("HTTP Server %s already shut down" % self.httpserver) + stop.priority = 25 + + def restart(self): + """Restart the HTTP server.""" + self.stop() + self.start() + + +class FlupCGIServer(object): + """Adapter for a flup.server.cgi.WSGIServer.""" + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.ready = False + + def start(self): + """Start the CGI server.""" + # We have to instantiate the server class here because its __init__ + # starts a threadpool. If we do it too early, daemonize won't work. + from flup.server.cgi import WSGIServer + + self.cgiserver = WSGIServer(*self.args, **self.kwargs) + self.ready = True + self.cgiserver.run() + + def stop(self): + """Stop the HTTP server.""" + self.ready = False + + +class FlupFCGIServer(object): + """Adapter for a flup.server.fcgi.WSGIServer.""" + + def __init__(self, *args, **kwargs): + if kwargs.get('bindAddress', None) is None: + import socket + if not hasattr(socket, 'fromfd'): + raise ValueError( + 'Dynamic FCGI server not available on this platform. ' + 'You must use a static or external one by providing a ' + 'legal bindAddress.') + self.args = args + self.kwargs = kwargs + self.ready = False + + def start(self): + """Start the FCGI server.""" + # We have to instantiate the server class here because its __init__ + # starts a threadpool. If we do it too early, daemonize won't work. + from flup.server.fcgi import WSGIServer + self.fcgiserver = WSGIServer(*self.args, **self.kwargs) + # TODO: report this bug upstream to flup. + # If we don't set _oldSIGs on Windows, we get: + # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", + # line 108, in run + # self._restoreSignalHandlers() + # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", + # line 156, in _restoreSignalHandlers + # for signum,handler in self._oldSIGs: + # AttributeError: 'WSGIServer' object has no attribute '_oldSIGs' + self.fcgiserver._installSignalHandlers = lambda: None + self.fcgiserver._oldSIGs = [] + self.ready = True + self.fcgiserver.run() + + def stop(self): + """Stop the HTTP server.""" + # Forcibly stop the fcgi server main event loop. + self.fcgiserver._keepGoing = False + # Force all worker threads to die off. + self.fcgiserver._threadPool.maxSpare = self.fcgiserver._threadPool._idleCount + self.ready = False + + +class FlupSCGIServer(object): + """Adapter for a flup.server.scgi.WSGIServer.""" + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.ready = False + + def start(self): + """Start the SCGI server.""" + # We have to instantiate the server class here because its __init__ + # starts a threadpool. If we do it too early, daemonize won't work. + from flup.server.scgi import WSGIServer + self.scgiserver = WSGIServer(*self.args, **self.kwargs) + # TODO: report this bug upstream to flup. + # If we don't set _oldSIGs on Windows, we get: + # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", + # line 108, in run + # self._restoreSignalHandlers() + # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", + # line 156, in _restoreSignalHandlers + # for signum,handler in self._oldSIGs: + # AttributeError: 'WSGIServer' object has no attribute '_oldSIGs' + self.scgiserver._installSignalHandlers = lambda: None + self.scgiserver._oldSIGs = [] + self.ready = True + self.scgiserver.run() + + def stop(self): + """Stop the HTTP server.""" + self.ready = False + # Forcibly stop the scgi server main event loop. + self.scgiserver._keepGoing = False + # Force all worker threads to die off. + self.scgiserver._threadPool.maxSpare = 0 + + +def client_host(server_host): + """Return the host on which a client can connect to the given listener.""" + if server_host == '0.0.0.0': + # 0.0.0.0 is INADDR_ANY, which should answer on localhost. + return '127.0.0.1' + if server_host in ('::', '::0', '::0.0.0.0'): + # :: is IN6ADDR_ANY, which should answer on localhost. + # ::0 and ::0.0.0.0 are non-canonical but common ways to write IN6ADDR_ANY. + return '::1' + return server_host + +def check_port(host, port, timeout=1.0): + """Raise an error if the given port is not free on the given host.""" + if not host: + raise ValueError("Host values of '' or None are not allowed.") + host = client_host(host) + port = int(port) + + import socket + + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 addresses) + try: + info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM) + except socket.gaierror: + if ':' in host: + info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))] + else: + info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] + + for res in info: + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(timeout) + s.connect((host, port)) + s.close() + raise IOError("Port %s is in use on %s; perhaps the previous " + "httpserver did not shut down properly." % + (repr(port), repr(host))) + except socket.error: + if s: + s.close() + +def wait_for_free_port(host, port): + """Wait for the specified port to become free (drop requests).""" + if not host: + raise ValueError("Host values of '' or None are not allowed.") + + for trial in range(50): + try: + # we are expecting a free port, so reduce the timeout + check_port(host, port, timeout=0.1) + except IOError: + # Give the old server thread time to free the port. + time.sleep(0.1) + else: + return + + raise IOError("Port %r not free on %r" % (port, host)) + +def wait_for_occupied_port(host, port): + """Wait for the specified port to become active (receive requests).""" + if not host: + raise ValueError("Host values of '' or None are not allowed.") + + for trial in range(50): + try: + check_port(host, port) + except IOError: + return + else: + time.sleep(.1) + + raise IOError("Port %r not bound on %r" % (port, host)) diff --git a/cherrypy/process/win32.py b/cherrypy/process/win32.py new file mode 100755 index 0000000..83f99a5 --- /dev/null +++ b/cherrypy/process/win32.py @@ -0,0 +1,174 @@ +"""Windows service. Requires pywin32.""" + +import os +import win32api +import win32con +import win32event +import win32service +import win32serviceutil + +from cherrypy.process import wspbus, plugins + + +class ConsoleCtrlHandler(plugins.SimplePlugin): + """A WSPBus plugin for handling Win32 console events (like Ctrl-C).""" + + def __init__(self, bus): + self.is_set = False + plugins.SimplePlugin.__init__(self, bus) + + def start(self): + if self.is_set: + self.bus.log('Handler for console events already set.', level=40) + return + + result = win32api.SetConsoleCtrlHandler(self.handle, 1) + if result == 0: + self.bus.log('Could not SetConsoleCtrlHandler (error %r)' % + win32api.GetLastError(), level=40) + else: + self.bus.log('Set handler for console events.', level=40) + self.is_set = True + + def stop(self): + if not self.is_set: + self.bus.log('Handler for console events already off.', level=40) + return + + try: + result = win32api.SetConsoleCtrlHandler(self.handle, 0) + except ValueError: + # "ValueError: The object has not been registered" + result = 1 + + if result == 0: + self.bus.log('Could not remove SetConsoleCtrlHandler (error %r)' % + win32api.GetLastError(), level=40) + else: + self.bus.log('Removed handler for console events.', level=40) + self.is_set = False + + def handle(self, event): + """Handle console control events (like Ctrl-C).""" + if event in (win32con.CTRL_C_EVENT, win32con.CTRL_LOGOFF_EVENT, + win32con.CTRL_BREAK_EVENT, win32con.CTRL_SHUTDOWN_EVENT, + win32con.CTRL_CLOSE_EVENT): + self.bus.log('Console event %s: shutting down bus' % event) + + # Remove self immediately so repeated Ctrl-C doesn't re-call it. + try: + self.stop() + except ValueError: + pass + + self.bus.exit() + # 'First to return True stops the calls' + return 1 + return 0 + + +class Win32Bus(wspbus.Bus): + """A Web Site Process Bus implementation for Win32. + + Instead of time.sleep, this bus blocks using native win32event objects. + """ + + def __init__(self): + self.events = {} + wspbus.Bus.__init__(self) + + def _get_state_event(self, state): + """Return a win32event for the given state (creating it if needed).""" + try: + return self.events[state] + except KeyError: + event = win32event.CreateEvent(None, 0, 0, + "WSPBus %s Event (pid=%r)" % + (state.name, os.getpid())) + self.events[state] = event + return event + + def _get_state(self): + return self._state + def _set_state(self, value): + self._state = value + event = self._get_state_event(value) + win32event.PulseEvent(event) + state = property(_get_state, _set_state) + + def wait(self, state, interval=0.1, channel=None): + """Wait for the given state(s), KeyboardInterrupt or SystemExit. + + Since this class uses native win32event objects, the interval + argument is ignored. + """ + if isinstance(state, (tuple, list)): + # Don't wait for an event that beat us to the punch ;) + if self.state not in state: + events = tuple([self._get_state_event(s) for s in state]) + win32event.WaitForMultipleObjects(events, 0, win32event.INFINITE) + else: + # Don't wait for an event that beat us to the punch ;) + if self.state != state: + event = self._get_state_event(state) + win32event.WaitForSingleObject(event, win32event.INFINITE) + + +class _ControlCodes(dict): + """Control codes used to "signal" a service via ControlService. + + User-defined control codes are in the range 128-255. We generally use + the standard Python value for the Linux signal and add 128. Example: + + >>> signal.SIGUSR1 + 10 + control_codes['graceful'] = 128 + 10 + """ + + def key_for(self, obj): + """For the given value, return its corresponding key.""" + for key, val in self.items(): + if val is obj: + return key + raise ValueError("The given object could not be found: %r" % obj) + +control_codes = _ControlCodes({'graceful': 138}) + + +def signal_child(service, command): + if command == 'stop': + win32serviceutil.StopService(service) + elif command == 'restart': + win32serviceutil.RestartService(service) + else: + win32serviceutil.ControlService(service, control_codes[command]) + + +class PyWebService(win32serviceutil.ServiceFramework): + """Python Web Service.""" + + _svc_name_ = "Python Web Service" + _svc_display_name_ = "Python Web Service" + _svc_deps_ = None # sequence of service names on which this depends + _exe_name_ = "pywebsvc" + _exe_args_ = None # Default to no arguments + + # Only exists on Windows 2000 or later, ignored on windows NT + _svc_description_ = "Python Web Service" + + def SvcDoRun(self): + from cherrypy import process + process.bus.start() + process.bus.block() + + def SvcStop(self): + from cherrypy import process + self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) + process.bus.exit() + + def SvcOther(self, control): + process.bus.publish(control_codes.key_for(control)) + + +if __name__ == '__main__': + win32serviceutil.HandleCommandLine(PyWebService) diff --git a/cherrypy/process/wspbus.py b/cherrypy/process/wspbus.py new file mode 100755 index 0000000..46cd75a --- /dev/null +++ b/cherrypy/process/wspbus.py @@ -0,0 +1,393 @@ +"""An implementation of the Web Site Process Bus. + +This module is completely standalone, depending only on the stdlib. + +Web Site Process Bus +-------------------- + +A Bus object is used to contain and manage site-wide behavior: +daemonization, HTTP server start/stop, process reload, signal handling, +drop privileges, PID file management, logging for all of these, +and many more. + +In addition, a Bus object provides a place for each web framework +to register code that runs in response to site-wide events (like +process start and stop), or which controls or otherwise interacts with +the site-wide components mentioned above. For example, a framework which +uses file-based templates would add known template filenames to an +autoreload component. + +Ideally, a Bus object will be flexible enough to be useful in a variety +of invocation scenarios: + + 1. The deployer starts a site from the command line via a + framework-neutral deployment script; applications from multiple frameworks + are mixed in a single site. Command-line arguments and configuration + files are used to define site-wide components such as the HTTP server, + WSGI component graph, autoreload behavior, signal handling, etc. + 2. The deployer starts a site via some other process, such as Apache; + applications from multiple frameworks are mixed in a single site. + Autoreload and signal handling (from Python at least) are disabled. + 3. The deployer starts a site via a framework-specific mechanism; + for example, when running tests, exploring tutorials, or deploying + single applications from a single framework. The framework controls + which site-wide components are enabled as it sees fit. + +The Bus object in this package uses topic-based publish-subscribe +messaging to accomplish all this. A few topic channels are built in +('start', 'stop', 'exit', 'graceful', 'log', and 'main'). Frameworks and +site containers are free to define their own. If a message is sent to a +channel that has not been defined or has no listeners, there is no effect. + +In general, there should only ever be a single Bus object per process. +Frameworks and site containers share a single Bus object by publishing +messages and subscribing listeners. + +The Bus object works as a finite state machine which models the current +state of the process. Bus methods move it from one state to another; +those methods then publish to subscribed listeners on the channel for +the new state.:: + + O + | + V + STOPPING --> STOPPED --> EXITING -> X + A A | + | \___ | + | \ | + | V V + STARTED <-- STARTING + +""" + +import atexit +import os +import sys +import threading +import time +import traceback as _traceback +import warnings + +from cherrypy._cpcompat import set + +# Here I save the value of os.getcwd(), which, if I am imported early enough, +# will be the directory from which the startup script was run. This is needed +# by _do_execv(), to change back to the original directory before execv()ing a +# new process. This is a defense against the application having changed the +# current working directory (which could make sys.executable "not found" if +# sys.executable is a relative-path, and/or cause other problems). +_startup_cwd = os.getcwd() + +class ChannelFailures(Exception): + """Exception raised when errors occur in a listener during Bus.publish().""" + delimiter = '\n' + + def __init__(self, *args, **kwargs): + # Don't use 'super' here; Exceptions are old-style in Py2.4 + # See http://www.cherrypy.org/ticket/959 + Exception.__init__(self, *args, **kwargs) + self._exceptions = list() + + def handle_exception(self): + """Append the current exception to self.""" + self._exceptions.append(sys.exc_info()) + + def get_instances(self): + """Return a list of seen exception instances.""" + return [instance for cls, instance, traceback in self._exceptions] + + def __str__(self): + exception_strings = map(repr, self.get_instances()) + return self.delimiter.join(exception_strings) + + __repr__ = __str__ + + def __nonzero__(self): + return bool(self._exceptions) + +# Use a flag to indicate the state of the bus. +class _StateEnum(object): + class State(object): + name = None + def __repr__(self): + return "states.%s" % self.name + + def __setattr__(self, key, value): + if isinstance(value, self.State): + value.name = key + object.__setattr__(self, key, value) +states = _StateEnum() +states.STOPPED = states.State() +states.STARTING = states.State() +states.STARTED = states.State() +states.STOPPING = states.State() +states.EXITING = states.State() + + +class Bus(object): + """Process state-machine and messenger for HTTP site deployment. + + All listeners for a given channel are guaranteed to be called even + if others at the same channel fail. Each failure is logged, but + execution proceeds on to the next listener. The only way to stop all + processing from inside a listener is to raise SystemExit and stop the + whole server. + """ + + states = states + state = states.STOPPED + execv = False + + def __init__(self): + self.execv = False + self.state = states.STOPPED + self.listeners = dict( + [(channel, set()) for channel + in ('start', 'stop', 'exit', 'graceful', 'log', 'main')]) + self._priorities = {} + + def subscribe(self, channel, callback, priority=None): + """Add the given callback at the given channel (if not present).""" + if channel not in self.listeners: + self.listeners[channel] = set() + self.listeners[channel].add(callback) + + if priority is None: + priority = getattr(callback, 'priority', 50) + self._priorities[(channel, callback)] = priority + + def unsubscribe(self, channel, callback): + """Discard the given callback (if present).""" + listeners = self.listeners.get(channel) + if listeners and callback in listeners: + listeners.discard(callback) + del self._priorities[(channel, callback)] + + def publish(self, channel, *args, **kwargs): + """Return output of all subscribers for the given channel.""" + if channel not in self.listeners: + return [] + + exc = ChannelFailures() + output = [] + + items = [(self._priorities[(channel, listener)], listener) + for listener in self.listeners[channel]] + items.sort() + for priority, listener in items: + try: + output.append(listener(*args, **kwargs)) + except KeyboardInterrupt: + raise + except SystemExit, e: + # If we have previous errors ensure the exit code is non-zero + if exc and e.code == 0: + e.code = 1 + raise + except: + exc.handle_exception() + if channel == 'log': + # Assume any further messages to 'log' will fail. + pass + else: + self.log("Error in %r listener %r" % (channel, listener), + level=40, traceback=True) + if exc: + raise exc + return output + + def _clean_exit(self): + """An atexit handler which asserts the Bus is not running.""" + if self.state != states.EXITING: + warnings.warn( + "The main thread is exiting, but the Bus is in the %r state; " + "shutting it down automatically now. You must either call " + "bus.block() after start(), or call bus.exit() before the " + "main thread exits." % self.state, RuntimeWarning) + self.exit() + + def start(self): + """Start all services.""" + atexit.register(self._clean_exit) + + self.state = states.STARTING + self.log('Bus STARTING') + try: + self.publish('start') + self.state = states.STARTED + self.log('Bus STARTED') + except (KeyboardInterrupt, SystemExit): + raise + except: + self.log("Shutting down due to error in start listener:", + level=40, traceback=True) + e_info = sys.exc_info() + try: + self.exit() + except: + # Any stop/exit errors will be logged inside publish(). + pass + raise e_info[0], e_info[1], e_info[2] + + def exit(self): + """Stop all services and prepare to exit the process.""" + exitstate = self.state + try: + self.stop() + + self.state = states.EXITING + self.log('Bus EXITING') + self.publish('exit') + # This isn't strictly necessary, but it's better than seeing + # "Waiting for child threads to terminate..." and then nothing. + self.log('Bus EXITED') + except: + # This method is often called asynchronously (whether thread, + # signal handler, console handler, or atexit handler), so we + # can't just let exceptions propagate out unhandled. + # Assume it's been logged and just die. + os._exit(70) # EX_SOFTWARE + + if exitstate == states.STARTING: + # exit() was called before start() finished, possibly due to + # Ctrl-C because a start listener got stuck. In this case, + # we could get stuck in a loop where Ctrl-C never exits the + # process, so we just call os.exit here. + os._exit(70) # EX_SOFTWARE + + def restart(self): + """Restart the process (may close connections). + + This method does not restart the process from the calling thread; + instead, it stops the bus and asks the main thread to call execv. + """ + self.execv = True + self.exit() + + def graceful(self): + """Advise all services to reload.""" + self.log('Bus graceful') + self.publish('graceful') + + def block(self, interval=0.1): + """Wait for the EXITING state, KeyboardInterrupt or SystemExit. + + This function is intended to be called only by the main thread. + After waiting for the EXITING state, it also waits for all threads + to terminate, and then calls os.execv if self.execv is True. This + design allows another thread to call bus.restart, yet have the main + thread perform the actual execv call (required on some platforms). + """ + try: + self.wait(states.EXITING, interval=interval, channel='main') + except (KeyboardInterrupt, IOError): + # The time.sleep call might raise + # "IOError: [Errno 4] Interrupted function call" on KBInt. + self.log('Keyboard Interrupt: shutting down bus') + self.exit() + except SystemExit: + self.log('SystemExit raised: shutting down bus') + self.exit() + raise + + # Waiting for ALL child threads to finish is necessary on OS X. + # See http://www.cherrypy.org/ticket/581. + # It's also good to let them all shut down before allowing + # the main thread to call atexit handlers. + # See http://www.cherrypy.org/ticket/751. + self.log("Waiting for child threads to terminate...") + for t in threading.enumerate(): + if t != threading.currentThread() and t.isAlive(): + # Note that any dummy (external) threads are always daemonic. + if hasattr(threading.Thread, "daemon"): + # Python 2.6+ + d = t.daemon + else: + d = t.isDaemon() + if not d: + self.log("Waiting for thread %s." % t.getName()) + t.join() + + if self.execv: + self._do_execv() + + def wait(self, state, interval=0.1, channel=None): + """Poll for the given state(s) at intervals; publish to channel.""" + if isinstance(state, (tuple, list)): + states = state + else: + states = [state] + + def _wait(): + while self.state not in states: + time.sleep(interval) + self.publish(channel) + + # From http://psyco.sourceforge.net/psycoguide/bugs.html: + # "The compiled machine code does not include the regular polling + # done by Python, meaning that a KeyboardInterrupt will not be + # detected before execution comes back to the regular Python + # interpreter. Your program cannot be interrupted if caught + # into an infinite Psyco-compiled loop." + try: + sys.modules['psyco'].cannotcompile(_wait) + except (KeyError, AttributeError): + pass + + _wait() + + def _do_execv(self): + """Re-execute the current process. + + This must be called from the main thread, because certain platforms + (OS X) don't allow execv to be called in a child thread very well. + """ + args = sys.argv[:] + self.log('Re-spawning %s' % ' '.join(args)) + + if sys.platform[:4] == 'java': + from _systemrestart import SystemRestart + raise SystemRestart + else: + args.insert(0, sys.executable) + if sys.platform == 'win32': + args = ['"%s"' % arg for arg in args] + + os.chdir(_startup_cwd) + os.execv(sys.executable, args) + + def stop(self): + """Stop all services.""" + self.state = states.STOPPING + self.log('Bus STOPPING') + self.publish('stop') + self.state = states.STOPPED + self.log('Bus STOPPED') + + def start_with_callback(self, func, args=None, kwargs=None): + """Start 'func' in a new thread T, then start self (and return T).""" + if args is None: + args = () + if kwargs is None: + kwargs = {} + args = (func,) + args + + def _callback(func, *a, **kw): + self.wait(states.STARTED) + func(*a, **kw) + t = threading.Thread(target=_callback, args=args, kwargs=kwargs) + t.setName('Bus Callback ' + t.getName()) + t.start() + + self.start() + + return t + + def log(self, msg="", level=20, traceback=False): + """Log the given message. Append the last traceback if requested.""" + if traceback: + exc = sys.exc_info() + msg += "\n" + "".join(_traceback.format_exception(*exc)) + self.publish('log', msg, level) + +bus = Bus() diff --git a/cherrypy/scaffold/__init__.py b/cherrypy/scaffold/__init__.py new file mode 100755 index 0000000..00964ac --- /dev/null +++ b/cherrypy/scaffold/__init__.py @@ -0,0 +1,61 @@ +"""<MyProject>, a CherryPy application. + +Use this as a base for creating new CherryPy applications. When you want +to make a new app, copy and paste this folder to some other location +(maybe site-packages) and rename it to the name of your project, +then tweak as desired. + +Even before any tweaking, this should serve a few demonstration pages. +Change to this directory and run: + + ../cherryd -c site.conf + +""" + +import cherrypy +from cherrypy import tools, url + +import os +local_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + + +class Root: + + _cp_config = {'tools.log_tracebacks.on': True, + } + + def index(self): + return """<html> +<body>Try some <a href='%s?a=7'>other</a> path, +or a <a href='%s?n=14'>default</a> path.<br /> +Or, just look at the pretty picture:<br /> +<img src='%s' /> +</body></html>""" % (url("other"), url("else"), + url("files/made_with_cherrypy_small.png")) + index.exposed = True + + def default(self, *args, **kwargs): + return "args: %s kwargs: %s" % (args, kwargs) + default.exposed = True + + def other(self, a=2, b='bananas', c=None): + cherrypy.response.headers['Content-Type'] = 'text/plain' + if c is None: + return "Have %d %s." % (int(a), b) + else: + return "Have %d %s, %s." % (int(a), b, c) + other.exposed = True + + files = cherrypy.tools.staticdir.handler( + section="/files", + dir=os.path.join(local_dir, "static"), + # Ignore .php files, etc. + match=r'\.(css|gif|html?|ico|jpe?g|js|png|swf|xml)$', + ) + + +root = Root() + +# Uncomment the following to use your own favicon instead of CP's default. +#favicon_path = os.path.join(local_dir, "favicon.ico") +#root.favicon_ico = tools.staticfile.handler(filename=favicon_path) diff --git a/cherrypy/scaffold/example.conf b/cherrypy/scaffold/example.conf new file mode 100644 index 0000000..93a6e53 --- /dev/null +++ b/cherrypy/scaffold/example.conf @@ -0,0 +1,3 @@ +[/] +log.error_file: "error.log" +log.access_file: "access.log"
\ No newline at end of file diff --git a/cherrypy/scaffold/site.conf b/cherrypy/scaffold/site.conf new file mode 100644 index 0000000..6ed3898 --- /dev/null +++ b/cherrypy/scaffold/site.conf @@ -0,0 +1,14 @@ +[global] +# Uncomment this when you're done developing +#environment: "production" + +server.socket_host: "0.0.0.0" +server.socket_port: 8088 + +# Uncomment the following lines to run on HTTPS at the same time +#server.2.socket_host: "0.0.0.0" +#server.2.socket_port: 8433 +#server.2.ssl_certificate: '../test/test.pem' +#server.2.ssl_private_key: '../test/test.pem' + +tree.myapp: cherrypy.Application(scaffold.root, "/", "example.conf") diff --git a/cherrypy/scaffold/static/made_with_cherrypy_small.png b/cherrypy/scaffold/static/made_with_cherrypy_small.png Binary files differnew file mode 100644 index 0000000..c3aafee --- /dev/null +++ b/cherrypy/scaffold/static/made_with_cherrypy_small.png diff --git a/cherrypy/test/__init__.py b/cherrypy/test/__init__.py new file mode 100755 index 0000000..e4c400d --- /dev/null +++ b/cherrypy/test/__init__.py @@ -0,0 +1,25 @@ +"""Regression test suite for CherryPy. + +Run 'nosetests -s test/' to exercise all tests. + +The '-s' flag instructs nose to output stdout messages, wihch is crucial to +the 'interactive' mode of webtest.py. If you run these tests without the '-s' +flag, don't be surprised if the test seems to hang: it's waiting for your +interactive input. +""" + +import sys +def newexit(): + raise SystemExit('Exit called') + +def setup(): + # We want to monkey patch sys.exit so that we can get some + # information about where exit is being called. + newexit._old = sys.exit + sys.exit = newexit + +def teardown(): + try: + sys.exit = sys.exit._old + except AttributeError: + sys.exit = sys._exit diff --git a/cherrypy/test/_test_decorators.py b/cherrypy/test/_test_decorators.py new file mode 100755 index 0000000..5bcbc1e --- /dev/null +++ b/cherrypy/test/_test_decorators.py @@ -0,0 +1,41 @@ +"""Test module for the @-decorator syntax, which is version-specific""" + +from cherrypy import expose, tools +from cherrypy._cpcompat import ntob + + +class ExposeExamples(object): + + @expose + def no_call(self): + return "Mr E. R. Bradshaw" + + @expose() + def call_empty(self): + return "Mrs. B.J. Smegma" + + @expose("call_alias") + def nesbitt(self): + return "Mr Nesbitt" + + @expose(["alias1", "alias2"]) + def andrews(self): + return "Mr Ken Andrews" + + @expose(alias="alias3") + def watson(self): + return "Mr. and Mrs. Watson" + + +class ToolExamples(object): + + @expose + @tools.response_headers(headers=[('Content-Type', 'application/data')]) + def blah(self): + yield ntob("blah") + # This is here to demonstrate that _cp_config = {...} overwrites + # the _cp_config attribute added by the Tool decorator. You have + # to write _cp_config[k] = v or _cp_config.update(...) instead. + blah._cp_config['response.stream'] = True + + diff --git a/cherrypy/test/_test_states_demo.py b/cherrypy/test/_test_states_demo.py new file mode 100755 index 0000000..3f8f196 --- /dev/null +++ b/cherrypy/test/_test_states_demo.py @@ -0,0 +1,66 @@ +import os +import sys +import time +starttime = time.time() + +import cherrypy + + +class Root: + + def index(self): + return "Hello World" + index.exposed = True + + def mtimes(self): + return repr(cherrypy.engine.publish("Autoreloader", "mtimes")) + mtimes.exposed = True + + def pid(self): + return str(os.getpid()) + pid.exposed = True + + def start(self): + return repr(starttime) + start.exposed = True + + def exit(self): + # This handler might be called before the engine is STARTED if an + # HTTP worker thread handles it before the HTTP server returns + # control to engine.start. We avoid that race condition here + # by waiting for the Bus to be STARTED. + cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) + cherrypy.engine.exit() + exit.exposed = True + + +def unsub_sig(): + cherrypy.log("unsubsig: %s" % cherrypy.config.get('unsubsig', False)) + if cherrypy.config.get('unsubsig', False): + cherrypy.log("Unsubscribing the default cherrypy signal handler") + cherrypy.engine.signal_handler.unsubscribe() + try: + from signal import signal, SIGTERM + except ImportError: + pass + else: + def old_term_handler(signum=None, frame=None): + cherrypy.log("I am an old SIGTERM handler.") + sys.exit(0) + cherrypy.log("Subscribing the new one.") + signal(SIGTERM, old_term_handler) +cherrypy.engine.subscribe('start', unsub_sig, priority=100) + + +def starterror(): + if cherrypy.config.get('starterror', False): + zerodiv = 1 / 0 +cherrypy.engine.subscribe('start', starterror, priority=6) + +def log_test_case_name(): + if cherrypy.config.get('test_case_name', False): + cherrypy.log("STARTED FROM: %s" % cherrypy.config.get('test_case_name')) +cherrypy.engine.subscribe('start', log_test_case_name, priority=6) + + +cherrypy.tree.mount(Root(), '/', {'/': {}}) diff --git a/cherrypy/test/benchmark.py b/cherrypy/test/benchmark.py new file mode 100755 index 0000000..90536a5 --- /dev/null +++ b/cherrypy/test/benchmark.py @@ -0,0 +1,409 @@ +"""CherryPy Benchmark Tool + + Usage: + benchmark.py --null --notests --help --cpmodpy --modpython --ab=path --apache=path + + --null: use a null Request object (to bench the HTTP server only) + --notests: start the server but do not run the tests; this allows + you to check the tested pages with a browser + --help: show this help message + --cpmodpy: run tests via apache on 8080 (with the builtin _cpmodpy) + --modpython: run tests via apache on 8080 (with modpython_gateway) + --ab=path: Use the ab script/executable at 'path' (see below) + --apache=path: Use the apache script/exe at 'path' (see below) + + To run the benchmarks, the Apache Benchmark tool "ab" must either be on + your system path, or specified via the --ab=path option. + + To run the modpython tests, the "apache" executable or script must be + on your system path, or provided via the --apache=path option. On some + platforms, "apache" may be called "apachectl" or "apache2ctl"--create + a symlink to them if needed. +""" + +import getopt +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + +import re +import sys +import time +import traceback + +import cherrypy +from cherrypy._cpcompat import ntob +from cherrypy import _cperror, _cpmodpy +from cherrypy.lib import httputil + + +AB_PATH = "" +APACHE_PATH = "apache" +SCRIPT_NAME = "/cpbench/users/rdelon/apps/blog" + +__all__ = ['ABSession', 'Root', 'print_report', + 'run_standard_benchmarks', 'safe_threads', + 'size_report', 'startup', 'thread_report', + ] + +size_cache = {} + +class Root: + + def index(self): + return """<html> +<head> + <title>CherryPy Benchmark</title> +</head> +<body> + <ul> + <li><a href="hello">Hello, world! (14 byte dynamic)</a></li> + <li><a href="static/index.html">Static file (14 bytes static)</a></li> + <li><form action="sizer">Response of length: + <input type='text' name='size' value='10' /></form> + </li> + </ul> +</body> +</html>""" + index.exposed = True + + def hello(self): + return "Hello, world\r\n" + hello.exposed = True + + def sizer(self, size): + resp = size_cache.get(size, None) + if resp is None: + size_cache[size] = resp = "X" * int(size) + return resp + sizer.exposed = True + + +cherrypy.config.update({ + 'log.error.file': '', + 'environment': 'production', + 'server.socket_host': '127.0.0.1', + 'server.socket_port': 8080, + 'server.max_request_header_size': 0, + 'server.max_request_body_size': 0, + 'engine.deadlock_poll_freq': 0, + }) + +# Cheat mode on ;) +del cherrypy.config['tools.log_tracebacks.on'] +del cherrypy.config['tools.log_headers.on'] +del cherrypy.config['tools.trailing_slash.on'] + +appconf = { + '/static': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.root': curdir, + }, + } +app = cherrypy.tree.mount(Root(), SCRIPT_NAME, appconf) + + +class NullRequest: + """A null HTTP request class, returning 200 and an empty body.""" + + def __init__(self, local, remote, scheme="http"): + pass + + def close(self): + pass + + def run(self, method, path, query_string, protocol, headers, rfile): + cherrypy.response.status = "200 OK" + cherrypy.response.header_list = [("Content-Type", 'text/html'), + ("Server", "Null CherryPy"), + ("Date", httputil.HTTPDate()), + ("Content-Length", "0"), + ] + cherrypy.response.body = [""] + return cherrypy.response + + +class NullResponse: + pass + + +class ABSession: + """A session of 'ab', the Apache HTTP server benchmarking tool. + +Example output from ab: + +This is ApacheBench, Version 2.0.40-dev <$Revision: 1.121.2.1 $> apache-2.0 +Copyright (c) 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ +Copyright (c) 1998-2002 The Apache Software Foundation, http://www.apache.org/ + +Benchmarking 127.0.0.1 (be patient) +Completed 100 requests +Completed 200 requests +Completed 300 requests +Completed 400 requests +Completed 500 requests +Completed 600 requests +Completed 700 requests +Completed 800 requests +Completed 900 requests + + +Server Software: CherryPy/3.1beta +Server Hostname: 127.0.0.1 +Server Port: 8080 + +Document Path: /static/index.html +Document Length: 14 bytes + +Concurrency Level: 10 +Time taken for tests: 9.643867 seconds +Complete requests: 1000 +Failed requests: 0 +Write errors: 0 +Total transferred: 189000 bytes +HTML transferred: 14000 bytes +Requests per second: 103.69 [#/sec] (mean) +Time per request: 96.439 [ms] (mean) +Time per request: 9.644 [ms] (mean, across all concurrent requests) +Transfer rate: 19.08 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 2.9 0 10 +Processing: 20 94 7.3 90 130 +Waiting: 0 43 28.1 40 100 +Total: 20 95 7.3 100 130 + +Percentage of the requests served within a certain time (ms) + 50% 100 + 66% 100 + 75% 100 + 80% 100 + 90% 100 + 95% 100 + 98% 100 + 99% 110 + 100% 130 (longest request) +Finished 1000 requests +""" + + parse_patterns = [('complete_requests', 'Completed', + ntob(r'^Complete requests:\s*(\d+)')), + ('failed_requests', 'Failed', + ntob(r'^Failed requests:\s*(\d+)')), + ('requests_per_second', 'req/sec', + ntob(r'^Requests per second:\s*([0-9.]+)')), + ('time_per_request_concurrent', 'msec/req', + ntob(r'^Time per request:\s*([0-9.]+).*concurrent requests\)$')), + ('transfer_rate', 'KB/sec', + ntob(r'^Transfer rate:\s*([0-9.]+)')), + ] + + def __init__(self, path=SCRIPT_NAME + "/hello", requests=1000, concurrency=10): + self.path = path + self.requests = requests + self.concurrency = concurrency + + def args(self): + port = cherrypy.server.socket_port + assert self.concurrency > 0 + assert self.requests > 0 + # Don't use "localhost". + # Cf http://mail.python.org/pipermail/python-win32/2008-March/007050.html + return ("-k -n %s -c %s http://127.0.0.1:%s%s" % + (self.requests, self.concurrency, port, self.path)) + + def run(self): + # Parse output of ab, setting attributes on self + try: + self.output = _cpmodpy.read_process(AB_PATH or "ab", self.args()) + except: + print(_cperror.format_exc()) + raise + + for attr, name, pattern in self.parse_patterns: + val = re.search(pattern, self.output, re.MULTILINE) + if val: + val = val.group(1) + setattr(self, attr, val) + else: + setattr(self, attr, None) + + +safe_threads = (25, 50, 100, 200, 400) +if sys.platform in ("win32",): + # For some reason, ab crashes with > 50 threads on my Win2k laptop. + safe_threads = (10, 20, 30, 40, 50) + + +def thread_report(path=SCRIPT_NAME + "/hello", concurrency=safe_threads): + sess = ABSession(path) + attrs, names, patterns = list(zip(*sess.parse_patterns)) + avg = dict.fromkeys(attrs, 0.0) + + yield ('threads',) + names + for c in concurrency: + sess.concurrency = c + sess.run() + row = [c] + for attr in attrs: + val = getattr(sess, attr) + if val is None: + print(sess.output) + row = None + break + val = float(val) + avg[attr] += float(val) + row.append(val) + if row: + yield row + + # Add a row of averages. + yield ["Average"] + [str(avg[attr] / len(concurrency)) for attr in attrs] + +def size_report(sizes=(10, 100, 1000, 10000, 100000, 100000000), + concurrency=50): + sess = ABSession(concurrency=concurrency) + attrs, names, patterns = list(zip(*sess.parse_patterns)) + yield ('bytes',) + names + for sz in sizes: + sess.path = "%s/sizer?size=%s" % (SCRIPT_NAME, sz) + sess.run() + yield [sz] + [getattr(sess, attr) for attr in attrs] + +def print_report(rows): + for row in rows: + print("") + for i, val in enumerate(row): + sys.stdout.write(str(val).rjust(10) + " | ") + print("") + + +def run_standard_benchmarks(): + print("") + print("Client Thread Report (1000 requests, 14 byte response body, " + "%s server threads):" % cherrypy.server.thread_pool) + print_report(thread_report()) + + print("") + print("Client Thread Report (1000 requests, 14 bytes via staticdir, " + "%s server threads):" % cherrypy.server.thread_pool) + print_report(thread_report("%s/static/index.html" % SCRIPT_NAME)) + + print("") + print("Size Report (1000 requests, 50 client threads, " + "%s server threads):" % cherrypy.server.thread_pool) + print_report(size_report()) + + +# modpython and other WSGI # + +def startup_modpython(req=None): + """Start the CherryPy app server in 'serverless' mode (for modpython/WSGI).""" + if cherrypy.engine.state == cherrypy._cpengine.STOPPED: + if req: + if "nullreq" in req.get_options(): + cherrypy.engine.request_class = NullRequest + cherrypy.engine.response_class = NullResponse + ab_opt = req.get_options().get("ab", "") + if ab_opt: + global AB_PATH + AB_PATH = ab_opt + cherrypy.engine.start() + if cherrypy.engine.state == cherrypy._cpengine.STARTING: + cherrypy.engine.wait() + return 0 # apache.OK + + +def run_modpython(use_wsgi=False): + print("Starting mod_python...") + pyopts = [] + + # Pass the null and ab=path options through Apache + if "--null" in opts: + pyopts.append(("nullreq", "")) + + if "--ab" in opts: + pyopts.append(("ab", opts["--ab"])) + + s = _cpmodpy.ModPythonServer + if use_wsgi: + pyopts.append(("wsgi.application", "cherrypy::tree")) + pyopts.append(("wsgi.startup", "cherrypy.test.benchmark::startup_modpython")) + handler = "modpython_gateway::handler" + s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH, handler=handler) + else: + pyopts.append(("cherrypy.setup", "cherrypy.test.benchmark::startup_modpython")) + s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH) + + try: + s.start() + run() + finally: + s.stop() + + + +if __name__ == '__main__': + longopts = ['cpmodpy', 'modpython', 'null', 'notests', + 'help', 'ab=', 'apache='] + try: + switches, args = getopt.getopt(sys.argv[1:], "", longopts) + opts = dict(switches) + except getopt.GetoptError: + print(__doc__) + sys.exit(2) + + if "--help" in opts: + print(__doc__) + sys.exit(0) + + if "--ab" in opts: + AB_PATH = opts['--ab'] + + if "--notests" in opts: + # Return without stopping the server, so that the pages + # can be tested from a standard web browser. + def run(): + port = cherrypy.server.socket_port + print("You may now open http://127.0.0.1:%s%s/" % + (port, SCRIPT_NAME)) + + if "--null" in opts: + print("Using null Request object") + else: + def run(): + end = time.time() - start + print("Started in %s seconds" % end) + if "--null" in opts: + print("\nUsing null Request object") + try: + try: + run_standard_benchmarks() + except: + print(_cperror.format_exc()) + raise + finally: + cherrypy.engine.exit() + + print("Starting CherryPy app server...") + + class NullWriter(object): + """Suppresses the printing of socket errors.""" + def write(self, data): + pass + sys.stderr = NullWriter() + + start = time.time() + + if "--cpmodpy" in opts: + run_modpython() + elif "--modpython" in opts: + run_modpython(use_wsgi=True) + else: + if "--null" in opts: + cherrypy.server.request_class = NullRequest + cherrypy.server.response_class = NullResponse + + cherrypy.engine.start_with_callback(run) + cherrypy.engine.block() diff --git a/cherrypy/test/checkerdemo.py b/cherrypy/test/checkerdemo.py new file mode 100755 index 0000000..32a7dee --- /dev/null +++ b/cherrypy/test/checkerdemo.py @@ -0,0 +1,47 @@ +"""Demonstration app for cherrypy.checker. + +This application is intentionally broken and badly designed. +To demonstrate the output of the CherryPy Checker, simply execute +this module. +""" + +import os +import cherrypy +thisdir = os.path.dirname(os.path.abspath(__file__)) + +class Root: + pass + +if __name__ == '__main__': + conf = {'/base': {'tools.staticdir.root': thisdir, + # Obsolete key. + 'throw_errors': True, + }, + # This entry should be OK. + '/base/static': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static'}, + # Warn on missing folder. + '/base/js': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'js'}, + # Warn on dir with an abs path even though we provide root. + '/base/static2': {'tools.staticdir.on': True, + 'tools.staticdir.dir': '/static'}, + # Warn on dir with a relative path with no root. + '/static3': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static'}, + # Warn on unknown namespace + '/unknown': {'toobles.gzip.on': True}, + # Warn special on cherrypy.<known ns>.* + '/cpknown': {'cherrypy.tools.encode.on': True}, + # Warn on mismatched types + '/conftype': {'request.show_tracebacks': 14}, + # Warn on unknown tool. + '/web': {'tools.unknown.on': True}, + # Warn on server.* in app config. + '/app1': {'server.socket_host': '0.0.0.0'}, + # Warn on 'localhost' + 'global': {'server.socket_host': 'localhost'}, + # Warn on '[name]' + '[/extra_brackets]': {}, + } + cherrypy.quickstart(Root(), config=conf) diff --git a/cherrypy/test/helper.py b/cherrypy/test/helper.py new file mode 100755 index 0000000..ff9e06c --- /dev/null +++ b/cherrypy/test/helper.py @@ -0,0 +1,476 @@ +"""A library of helper functions for the CherryPy test suite.""" + +import datetime +import logging +log = logging.getLogger(__name__) +import os +thisdir = os.path.abspath(os.path.dirname(__file__)) +serverpem = os.path.join(os.getcwd(), thisdir, 'test.pem') + +import re +import sys +import time +import warnings + +import cherrypy +from cherrypy._cpcompat import basestring, copyitems, HTTPSConnection, ntob +from cherrypy.lib import httputil +from cherrypy.lib.reprconf import unrepr +from cherrypy.test import webtest + +import nose + +_testconfig = None + +def get_tst_config(overconf = {}): + global _testconfig + if _testconfig is None: + conf = { + 'scheme': 'http', + 'protocol': "HTTP/1.1", + 'port': 8080, + 'host': '127.0.0.1', + 'validate': False, + 'conquer': False, + 'server': 'wsgi', + } + try: + import testconfig + _conf = testconfig.config.get('supervisor', None) + if _conf is not None: + for k, v in _conf.items(): + if isinstance(v, basestring): + _conf[k] = unrepr(v) + conf.update(_conf) + except ImportError: + pass + _testconfig = conf + conf = _testconfig.copy() + conf.update(overconf) + + return conf + +class Supervisor(object): + """Base class for modeling and controlling servers during testing.""" + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + if k == 'port': + setattr(self, k, int(v)) + setattr(self, k, v) + + +log_to_stderr = lambda msg, level: sys.stderr.write(msg + os.linesep) + +class LocalSupervisor(Supervisor): + """Base class for modeling/controlling servers which run in the same process. + + When the server side runs in a different process, start/stop can dump all + state between each test module easily. When the server side runs in the + same process as the client, however, we have to do a bit more work to ensure + config and mounted apps are reset between tests. + """ + + using_apache = False + using_wsgi = False + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + cherrypy.server.httpserver = self.httpserver_class + + engine = cherrypy.engine + if hasattr(engine, "signal_handler"): + engine.signal_handler.subscribe() + if hasattr(engine, "console_control_handler"): + engine.console_control_handler.subscribe() + #engine.subscribe('log', log_to_stderr) + + def start(self, modulename=None): + """Load and start the HTTP server.""" + if modulename: + # Unhook httpserver so cherrypy.server.start() creates a new + # one (with config from setup_server, if declared). + cherrypy.server.httpserver = None + + cherrypy.engine.start() + + self.sync_apps() + + def sync_apps(self): + """Tell the server about any apps which the setup functions mounted.""" + pass + + def stop(self): + td = getattr(self, 'teardown', None) + if td: + td() + + cherrypy.engine.exit() + + for name, server in copyitems(getattr(cherrypy, 'servers', {})): + server.unsubscribe() + del cherrypy.servers[name] + + +class NativeServerSupervisor(LocalSupervisor): + """Server supervisor for the builtin HTTP server.""" + + httpserver_class = "cherrypy._cpnative_server.CPHTTPServer" + using_apache = False + using_wsgi = False + + def __str__(self): + return "Builtin HTTP Server on %s:%s" % (self.host, self.port) + + +class LocalWSGISupervisor(LocalSupervisor): + """Server supervisor for the builtin WSGI server.""" + + httpserver_class = "cherrypy._cpwsgi_server.CPWSGIServer" + using_apache = False + using_wsgi = True + + def __str__(self): + return "Builtin WSGI Server on %s:%s" % (self.host, self.port) + + def sync_apps(self): + """Hook a new WSGI app into the origin server.""" + cherrypy.server.httpserver.wsgi_app = self.get_app() + + def get_app(self, app=None): + """Obtain a new (decorated) WSGI app to hook into the origin server.""" + if app is None: + app = cherrypy.tree + + if self.conquer: + try: + import wsgiconq + except ImportError: + warnings.warn("Error importing wsgiconq. pyconquer will not run.") + else: + app = wsgiconq.WSGILogger(app, c_calls=True) + + if self.validate: + try: + from wsgiref import validate + except ImportError: + warnings.warn("Error importing wsgiref. The validator will not run.") + else: + #wraps the app in the validator + app = validate.validator(app) + + return app + + +def get_cpmodpy_supervisor(**options): + from cherrypy.test import modpy + sup = modpy.ModPythonSupervisor(**options) + sup.template = modpy.conf_cpmodpy + return sup + +def get_modpygw_supervisor(**options): + from cherrypy.test import modpy + sup = modpy.ModPythonSupervisor(**options) + sup.template = modpy.conf_modpython_gateway + sup.using_wsgi = True + return sup + +def get_modwsgi_supervisor(**options): + from cherrypy.test import modwsgi + return modwsgi.ModWSGISupervisor(**options) + +def get_modfcgid_supervisor(**options): + from cherrypy.test import modfcgid + return modfcgid.ModFCGISupervisor(**options) + +def get_modfastcgi_supervisor(**options): + from cherrypy.test import modfastcgi + return modfastcgi.ModFCGISupervisor(**options) + +def get_wsgi_u_supervisor(**options): + cherrypy.server.wsgi_version = ('u', 0) + return LocalWSGISupervisor(**options) + + +class CPWebCase(webtest.WebCase): + + script_name = "" + scheme = "http" + + available_servers = {'wsgi': LocalWSGISupervisor, + 'wsgi_u': get_wsgi_u_supervisor, + 'native': NativeServerSupervisor, + 'cpmodpy': get_cpmodpy_supervisor, + 'modpygw': get_modpygw_supervisor, + 'modwsgi': get_modwsgi_supervisor, + 'modfcgid': get_modfcgid_supervisor, + 'modfastcgi': get_modfastcgi_supervisor, + } + default_server = "wsgi" + + def _setup_server(cls, supervisor, conf): + v = sys.version.split()[0] + log.info("Python version used to run this test script: %s" % v) + log.info("CherryPy version: %s" % cherrypy.__version__) + if supervisor.scheme == "https": + ssl = " (ssl)" + else: + ssl = "" + log.info("HTTP server version: %s%s" % (supervisor.protocol, ssl)) + log.info("PID: %s" % os.getpid()) + + cherrypy.server.using_apache = supervisor.using_apache + cherrypy.server.using_wsgi = supervisor.using_wsgi + + if sys.platform[:4] == 'java': + cherrypy.config.update({'server.nodelay': False}) + + if isinstance(conf, basestring): + parser = cherrypy.lib.reprconf.Parser() + conf = parser.dict_from_file(conf).get('global', {}) + else: + conf = conf or {} + baseconf = conf.copy() + baseconf.update({'server.socket_host': supervisor.host, + 'server.socket_port': supervisor.port, + 'server.protocol_version': supervisor.protocol, + 'environment': "test_suite", + }) + if supervisor.scheme == "https": + #baseconf['server.ssl_module'] = 'builtin' + baseconf['server.ssl_certificate'] = serverpem + baseconf['server.ssl_private_key'] = serverpem + + # helper must be imported lazily so the coverage tool + # can run against module-level statements within cherrypy. + # Also, we have to do "from cherrypy.test import helper", + # exactly like each test module does, because a relative import + # would stick a second instance of webtest in sys.modules, + # and we wouldn't be able to globally override the port anymore. + if supervisor.scheme == "https": + webtest.WebCase.HTTP_CONN = HTTPSConnection + return baseconf + _setup_server = classmethod(_setup_server) + + def setup_class(cls): + '' + #Creates a server + conf = get_tst_config() + supervisor_factory = cls.available_servers.get(conf.get('server', 'wsgi')) + if supervisor_factory is None: + raise RuntimeError('Unknown server in config: %s' % conf['server']) + supervisor = supervisor_factory(**conf) + + #Copied from "run_test_suite" + cherrypy.config.reset() + baseconf = cls._setup_server(supervisor, conf) + cherrypy.config.update(baseconf) + setup_client() + + if hasattr(cls, 'setup_server'): + # Clear the cherrypy tree and clear the wsgi server so that + # it can be updated with the new root + cherrypy.tree = cherrypy._cptree.Tree() + cherrypy.server.httpserver = None + cls.setup_server() + supervisor.start(cls.__module__) + + cls.supervisor = supervisor + setup_class = classmethod(setup_class) + + def teardown_class(cls): + '' + if hasattr(cls, 'setup_server'): + cls.supervisor.stop() + teardown_class = classmethod(teardown_class) + + def prefix(self): + return self.script_name.rstrip("/") + + def base(self): + if ((self.scheme == "http" and self.PORT == 80) or + (self.scheme == "https" and self.PORT == 443)): + port = "" + else: + port = ":%s" % self.PORT + + return "%s://%s%s%s" % (self.scheme, self.HOST, port, + self.script_name.rstrip("/")) + + def exit(self): + sys.exit() + + def getPage(self, url, headers=None, method="GET", body=None, protocol=None): + """Open the url. Return status, headers, body.""" + if self.script_name: + url = httputil.urljoin(self.script_name, url) + return webtest.WebCase.getPage(self, url, headers, method, body, protocol) + + def skip(self, msg='skipped '): + raise nose.SkipTest(msg) + + def assertErrorPage(self, status, message=None, pattern=''): + """Compare the response body with a built in error page. + + The function will optionally look for the regexp pattern, + within the exception embedded in the error page.""" + + # This will never contain a traceback + page = cherrypy._cperror.get_error_page(status, message=message) + + # First, test the response body without checking the traceback. + # Stick a match-all group (.*) in to grab the traceback. + esc = re.escape + epage = esc(page) + epage = epage.replace(esc('<pre id="traceback"></pre>'), + esc('<pre id="traceback">') + '(.*)' + esc('</pre>')) + m = re.match(ntob(epage, self.encoding), self.body, re.DOTALL) + if not m: + self._handlewebError('Error page does not match; expected:\n' + page) + return + + # Now test the pattern against the traceback + if pattern is None: + # Special-case None to mean that there should be *no* traceback. + if m and m.group(1): + self._handlewebError('Error page contains traceback') + else: + if (m is None) or ( + not re.search(ntob(re.escape(pattern), self.encoding), + m.group(1))): + msg = 'Error page does not contain %s in traceback' + self._handlewebError(msg % repr(pattern)) + + date_tolerance = 2 + + def assertEqualDates(self, dt1, dt2, seconds=None): + """Assert abs(dt1 - dt2) is within Y seconds.""" + if seconds is None: + seconds = self.date_tolerance + + if dt1 > dt2: + diff = dt1 - dt2 + else: + diff = dt2 - dt1 + if not diff < datetime.timedelta(seconds=seconds): + raise AssertionError('%r and %r are not within %r seconds.' % + (dt1, dt2, seconds)) + + +def setup_client(): + """Set up the WebCase classes to match the server's socket settings.""" + webtest.WebCase.PORT = cherrypy.server.socket_port + webtest.WebCase.HOST = cherrypy.server.socket_host + if cherrypy.server.ssl_certificate: + CPWebCase.scheme = 'https' + +# --------------------------- Spawning helpers --------------------------- # + + +class CPProcess(object): + + pid_file = os.path.join(thisdir, 'test.pid') + config_file = os.path.join(thisdir, 'test.conf') + config_template = """[global] +server.socket_host: '%(host)s' +server.socket_port: %(port)s +checker.on: False +log.screen: False +log.error_file: r'%(error_log)s' +log.access_file: r'%(access_log)s' +%(ssl)s +%(extra)s +""" + error_log = os.path.join(thisdir, 'test.error.log') + access_log = os.path.join(thisdir, 'test.access.log') + + def __init__(self, wait=False, daemonize=False, ssl=False, socket_host=None, socket_port=None): + self.wait = wait + self.daemonize = daemonize + self.ssl = ssl + self.host = socket_host or cherrypy.server.socket_host + self.port = socket_port or cherrypy.server.socket_port + + def write_conf(self, extra=""): + if self.ssl: + serverpem = os.path.join(thisdir, 'test.pem') + ssl = """ +server.ssl_certificate: r'%s' +server.ssl_private_key: r'%s' +""" % (serverpem, serverpem) + else: + ssl = "" + + conf = self.config_template % { + 'host': self.host, + 'port': self.port, + 'error_log': self.error_log, + 'access_log': self.access_log, + 'ssl': ssl, + 'extra': extra, + } + f = open(self.config_file, 'wb') + f.write(ntob(conf, 'utf-8')) + f.close() + + def start(self, imports=None): + """Start cherryd in a subprocess.""" + cherrypy._cpserver.wait_for_free_port(self.host, self.port) + + args = [sys.executable, os.path.join(thisdir, '..', 'cherryd'), + '-c', self.config_file, '-p', self.pid_file] + + if not isinstance(imports, (list, tuple)): + imports = [imports] + for i in imports: + if i: + args.append('-i') + args.append(i) + + if self.daemonize: + args.append('-d') + + env = os.environ.copy() + # Make sure we import the cherrypy package in which this module is defined. + grandparentdir = os.path.abspath(os.path.join(thisdir, '..', '..')) + if env.get('PYTHONPATH', ''): + env['PYTHONPATH'] = os.pathsep.join((grandparentdir, env['PYTHONPATH'])) + else: + env['PYTHONPATH'] = grandparentdir + if self.wait: + self.exit_code = os.spawnve(os.P_WAIT, sys.executable, args, env) + else: + os.spawnve(os.P_NOWAIT, sys.executable, args, env) + cherrypy._cpserver.wait_for_occupied_port(self.host, self.port) + + # Give the engine a wee bit more time to finish STARTING + if self.daemonize: + time.sleep(2) + else: + time.sleep(1) + + def get_pid(self): + return int(open(self.pid_file, 'rb').read()) + + def join(self): + """Wait for the process to exit.""" + try: + try: + # Mac, UNIX + os.wait() + except AttributeError: + # Windows + try: + pid = self.get_pid() + except IOError: + # Assume the subprocess deleted the pidfile on shutdown. + pass + else: + os.waitpid(pid, 0) + except OSError: + x = sys.exc_info()[1] + if x.args != (10, 'No child processes'): + raise + diff --git a/cherrypy/test/logtest.py b/cherrypy/test/logtest.py new file mode 100755 index 0000000..c093da2 --- /dev/null +++ b/cherrypy/test/logtest.py @@ -0,0 +1,181 @@ +"""logtest, a unittest.TestCase helper for testing log output.""" + +import sys +import time + +import cherrypy + + +try: + # On Windows, msvcrt.getch reads a single char without output. + import msvcrt + def getchar(): + return msvcrt.getch() +except ImportError: + # Unix getchr + import tty, termios + def getchar(): + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + +class LogCase(object): + """unittest.TestCase mixin for testing log messages. + + logfile: a filename for the desired log. Yes, I know modes are evil, + but it makes the test functions so much cleaner to set this once. + + lastmarker: the last marker in the log. This can be used to search for + messages since the last marker. + + markerPrefix: a string with which to prefix log markers. This should be + unique enough from normal log output to use for marker identification. + """ + + logfile = None + lastmarker = None + markerPrefix = "test suite marker: " + + def _handleLogError(self, msg, data, marker, pattern): + print("") + print(" ERROR: %s" % msg) + + if not self.interactive: + raise self.failureException(msg) + + p = " Show: [L]og [M]arker [P]attern; [I]gnore, [R]aise, or sys.e[X]it >> " + print p, + # ARGH + sys.stdout.flush() + while True: + i = getchar().upper() + if i not in "MPLIRX": + continue + print(i.upper()) # Also prints new line + if i == "L": + for x, line in enumerate(data): + if (x + 1) % self.console_height == 0: + # The \r and comma should make the next line overwrite + print "<-- More -->\r", + m = getchar().lower() + # Erase our "More" prompt + print " \r", + if m == "q": + break + print(line.rstrip()) + elif i == "M": + print(repr(marker or self.lastmarker)) + elif i == "P": + print(repr(pattern)) + elif i == "I": + # return without raising the normal exception + return + elif i == "R": + raise self.failureException(msg) + elif i == "X": + self.exit() + print p, + + def exit(self): + sys.exit() + + def emptyLog(self): + """Overwrite self.logfile with 0 bytes.""" + open(self.logfile, 'wb').write("") + + def markLog(self, key=None): + """Insert a marker line into the log and set self.lastmarker.""" + if key is None: + key = str(time.time()) + self.lastmarker = key + + open(self.logfile, 'ab+').write("%s%s\n" % (self.markerPrefix, key)) + + def _read_marked_region(self, marker=None): + """Return lines from self.logfile in the marked region. + + If marker is None, self.lastmarker is used. If the log hasn't + been marked (using self.markLog), the entire log will be returned. + """ +## # Give the logger time to finish writing? +## time.sleep(0.5) + + logfile = self.logfile + marker = marker or self.lastmarker + if marker is None: + return open(logfile, 'rb').readlines() + + data = [] + in_region = False + for line in open(logfile, 'rb'): + if in_region: + if (line.startswith(self.markerPrefix) and not marker in line): + break + else: + data.append(line) + elif marker in line: + in_region = True + return data + + def assertInLog(self, line, marker=None): + """Fail if the given (partial) line is not in the log. + + The log will be searched from the given marker to the next marker. + If marker is None, self.lastmarker is used. If the log hasn't + been marked (using self.markLog), the entire log will be searched. + """ + data = self._read_marked_region(marker) + for logline in data: + if line in logline: + return + msg = "%r not found in log" % line + self._handleLogError(msg, data, marker, line) + + def assertNotInLog(self, line, marker=None): + """Fail if the given (partial) line is in the log. + + The log will be searched from the given marker to the next marker. + If marker is None, self.lastmarker is used. If the log hasn't + been marked (using self.markLog), the entire log will be searched. + """ + data = self._read_marked_region(marker) + for logline in data: + if line in logline: + msg = "%r found in log" % line + self._handleLogError(msg, data, marker, line) + + def assertLog(self, sliceargs, lines, marker=None): + """Fail if log.readlines()[sliceargs] is not contained in 'lines'. + + The log will be searched from the given marker to the next marker. + If marker is None, self.lastmarker is used. If the log hasn't + been marked (using self.markLog), the entire log will be searched. + """ + data = self._read_marked_region(marker) + if isinstance(sliceargs, int): + # Single arg. Use __getitem__ and allow lines to be str or list. + if isinstance(lines, (tuple, list)): + lines = lines[0] + if lines not in data[sliceargs]: + msg = "%r not found on log line %r" % (lines, sliceargs) + self._handleLogError(msg, [data[sliceargs]], marker, lines) + else: + # Multiple args. Use __getslice__ and require lines to be list. + if isinstance(lines, tuple): + lines = list(lines) + elif isinstance(lines, basestring): + raise TypeError("The 'lines' arg must be a list when " + "'sliceargs' is a tuple.") + + start, stop = sliceargs + for line, logline in zip(lines, data[start:stop]): + if line not in logline: + msg = "%r not found in log" % line + self._handleLogError(msg, data[start:stop], marker, line) + diff --git a/cherrypy/test/modfastcgi.py b/cherrypy/test/modfastcgi.py new file mode 100755 index 0000000..95acf14 --- /dev/null +++ b/cherrypy/test/modfastcgi.py @@ -0,0 +1,135 @@ +"""Wrapper for mod_fastcgi, for use as a CherryPy HTTP server when testing. + +To autostart fastcgi, the "apache" executable or script must be +on your system path, or you must override the global APACHE_PATH. +On some platforms, "apache" may be called "apachectl", "apache2ctl", +or "httpd"--create a symlink to them if needed. + +You'll also need the WSGIServer from flup.servers. +See http://projects.amor.org/misc/wiki/ModPythonGateway + + +KNOWN BUGS +========== + +1. Apache processes Range headers automatically; CherryPy's truncated + output is then truncated again by Apache. See test_core.testRanges. + This was worked around in http://www.cherrypy.org/changeset/1319. +2. Apache does not allow custom HTTP methods like CONNECT as per the spec. + See test_core.testHTTPMethods. +3. Max request header and body settings do not work with Apache. +4. Apache replaces status "reason phrases" automatically. For example, + CherryPy may set "304 Not modified" but Apache will write out + "304 Not Modified" (capital "M"). +5. Apache does not allow custom error codes as per the spec. +6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the + Request-URI too early. +7. mod_python will not read request bodies which use the "chunked" + transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block + instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and + mod_python's requestobject.c). +8. Apache will output a "Content-Length: 0" response header even if there's + no response entity body. This isn't really a bug; it just differs from + the CherryPy default. +""" + +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +import re +import sys +import time + +import cherrypy +from cherrypy.process import plugins, servers +from cherrypy.test import helper + + +def read_process(cmd, args=""): + pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + try: + firstline = pipeout.readline() + if (re.search(r"(not recognized|No such file|not found)", firstline, + re.IGNORECASE)): + raise IOError('%s must be on your system path.' % cmd) + output = firstline + pipeout.read() + finally: + pipeout.close() + return output + + +APACHE_PATH = "apache2ctl" +CONF_PATH = "fastcgi.conf" + +conf_fastcgi = """ +# Apache2 server conf file for testing CherryPy with mod_fastcgi. +# fumanchu: I had to hard-code paths due to crazy Debian layouts :( +ServerRoot /usr/lib/apache2 +User #1000 +ErrorLog %(root)s/mod_fastcgi.error.log + +DocumentRoot "%(root)s" +ServerName 127.0.0.1 +Listen %(port)s +LoadModule fastcgi_module modules/mod_fastcgi.so +LoadModule rewrite_module modules/mod_rewrite.so + +Options +ExecCGI +SetHandler fastcgi-script +RewriteEngine On +RewriteRule ^(.*)$ /fastcgi.pyc [L] +FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 +""" + +def erase_script_name(environ, start_response): + environ['SCRIPT_NAME'] = '' + return cherrypy.tree(environ, start_response) + +class ModFCGISupervisor(helper.LocalWSGISupervisor): + + httpserver_class = "cherrypy.process.servers.FlupFCGIServer" + using_apache = True + using_wsgi = True + template = conf_fastcgi + + def __str__(self): + return "FCGI Server on %s:%s" % (self.host, self.port) + + def start(self, modulename): + cherrypy.server.httpserver = servers.FlupFCGIServer( + application=erase_script_name, bindAddress=('127.0.0.1', 4000)) + cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) + cherrypy.server.socket_port = 4000 + # For FCGI, we both start apache... + self.start_apache() + # ...and our local server + cherrypy.engine.start() + self.sync_apps() + + def start_apache(self): + fcgiconf = CONF_PATH + if not os.path.isabs(fcgiconf): + fcgiconf = os.path.join(curdir, fcgiconf) + + # Write the Apache conf file. + f = open(fcgiconf, 'wb') + try: + server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] + output = self.template % {'port': self.port, 'root': curdir, + 'server': server} + output = output.replace('\r\n', '\n') + f.write(output) + finally: + f.close() + + result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) + if result: + print(result) + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + read_process(APACHE_PATH, "-k stop") + helper.LocalWSGISupervisor.stop(self) + + def sync_apps(self): + cherrypy.server.httpserver.fcgiserver.application = self.get_app(erase_script_name) + diff --git a/cherrypy/test/modfcgid.py b/cherrypy/test/modfcgid.py new file mode 100755 index 0000000..736aa4c --- /dev/null +++ b/cherrypy/test/modfcgid.py @@ -0,0 +1,125 @@ +"""Wrapper for mod_fcgid, for use as a CherryPy HTTP server when testing. + +To autostart fcgid, the "apache" executable or script must be +on your system path, or you must override the global APACHE_PATH. +On some platforms, "apache" may be called "apachectl", "apache2ctl", +or "httpd"--create a symlink to them if needed. + +You'll also need the WSGIServer from flup.servers. +See http://projects.amor.org/misc/wiki/ModPythonGateway + + +KNOWN BUGS +========== + +1. Apache processes Range headers automatically; CherryPy's truncated + output is then truncated again by Apache. See test_core.testRanges. + This was worked around in http://www.cherrypy.org/changeset/1319. +2. Apache does not allow custom HTTP methods like CONNECT as per the spec. + See test_core.testHTTPMethods. +3. Max request header and body settings do not work with Apache. +4. Apache replaces status "reason phrases" automatically. For example, + CherryPy may set "304 Not modified" but Apache will write out + "304 Not Modified" (capital "M"). +5. Apache does not allow custom error codes as per the spec. +6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the + Request-URI too early. +7. mod_python will not read request bodies which use the "chunked" + transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block + instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and + mod_python's requestobject.c). +8. Apache will output a "Content-Length: 0" response header even if there's + no response entity body. This isn't really a bug; it just differs from + the CherryPy default. +""" + +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +import re +import sys +import time + +import cherrypy +from cherrypy._cpcompat import ntob +from cherrypy.process import plugins, servers +from cherrypy.test import helper + + +def read_process(cmd, args=""): + pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + try: + firstline = pipeout.readline() + if (re.search(r"(not recognized|No such file|not found)", firstline, + re.IGNORECASE)): + raise IOError('%s must be on your system path.' % cmd) + output = firstline + pipeout.read() + finally: + pipeout.close() + return output + + +APACHE_PATH = "httpd" +CONF_PATH = "fcgi.conf" + +conf_fcgid = """ +# Apache2 server conf file for testing CherryPy with mod_fcgid. + +DocumentRoot "%(root)s" +ServerName 127.0.0.1 +Listen %(port)s +LoadModule fastcgi_module modules/mod_fastcgi.dll +LoadModule rewrite_module modules/mod_rewrite.so + +Options ExecCGI +SetHandler fastcgi-script +RewriteEngine On +RewriteRule ^(.*)$ /fastcgi.pyc [L] +FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 +""" + +class ModFCGISupervisor(helper.LocalSupervisor): + + using_apache = True + using_wsgi = True + template = conf_fcgid + + def __str__(self): + return "FCGI Server on %s:%s" % (self.host, self.port) + + def start(self, modulename): + cherrypy.server.httpserver = servers.FlupFCGIServer( + application=cherrypy.tree, bindAddress=('127.0.0.1', 4000)) + cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) + # For FCGI, we both start apache... + self.start_apache() + # ...and our local server + helper.LocalServer.start(self, modulename) + + def start_apache(self): + fcgiconf = CONF_PATH + if not os.path.isabs(fcgiconf): + fcgiconf = os.path.join(curdir, fcgiconf) + + # Write the Apache conf file. + f = open(fcgiconf, 'wb') + try: + server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] + output = self.template % {'port': self.port, 'root': curdir, + 'server': server} + output = ntob(output.replace('\r\n', '\n')) + f.write(output) + finally: + f.close() + + result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) + if result: + print(result) + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + read_process(APACHE_PATH, "-k stop") + helper.LocalServer.stop(self) + + def sync_apps(self): + cherrypy.server.httpserver.fcgiserver.application = self.get_app() + diff --git a/cherrypy/test/modpy.py b/cherrypy/test/modpy.py new file mode 100755 index 0000000..519571f --- /dev/null +++ b/cherrypy/test/modpy.py @@ -0,0 +1,163 @@ +"""Wrapper for mod_python, for use as a CherryPy HTTP server when testing. + +To autostart modpython, the "apache" executable or script must be +on your system path, or you must override the global APACHE_PATH. +On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- +create a symlink to them if needed. + +If you wish to test the WSGI interface instead of our _cpmodpy interface, +you also need the 'modpython_gateway' module at: +http://projects.amor.org/misc/wiki/ModPythonGateway + + +KNOWN BUGS +========== + +1. Apache processes Range headers automatically; CherryPy's truncated + output is then truncated again by Apache. See test_core.testRanges. + This was worked around in http://www.cherrypy.org/changeset/1319. +2. Apache does not allow custom HTTP methods like CONNECT as per the spec. + See test_core.testHTTPMethods. +3. Max request header and body settings do not work with Apache. +4. Apache replaces status "reason phrases" automatically. For example, + CherryPy may set "304 Not modified" but Apache will write out + "304 Not Modified" (capital "M"). +5. Apache does not allow custom error codes as per the spec. +6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the + Request-URI too early. +7. mod_python will not read request bodies which use the "chunked" + transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block + instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and + mod_python's requestobject.c). +8. Apache will output a "Content-Length: 0" response header even if there's + no response entity body. This isn't really a bug; it just differs from + the CherryPy default. +""" + +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +import re +import time + +from cherrypy.test import helper + + +def read_process(cmd, args=""): + pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + try: + firstline = pipeout.readline() + if (re.search(r"(not recognized|No such file|not found)", firstline, + re.IGNORECASE)): + raise IOError('%s must be on your system path.' % cmd) + output = firstline + pipeout.read() + finally: + pipeout.close() + return output + + +APACHE_PATH = "httpd" +CONF_PATH = "test_mp.conf" + +conf_modpython_gateway = """ +# Apache2 server conf file for testing CherryPy with modpython_gateway. + +ServerName 127.0.0.1 +DocumentRoot "/" +Listen %(port)s +LoadModule python_module modules/mod_python.so + +SetHandler python-program +PythonFixupHandler cherrypy.test.modpy::wsgisetup +PythonOption testmod %(modulename)s +PythonHandler modpython_gateway::handler +PythonOption wsgi.application cherrypy::tree +PythonOption socket_host %(host)s +PythonDebug On +""" + +conf_cpmodpy = """ +# Apache2 server conf file for testing CherryPy with _cpmodpy. + +ServerName 127.0.0.1 +DocumentRoot "/" +Listen %(port)s +LoadModule python_module modules/mod_python.so + +SetHandler python-program +PythonFixupHandler cherrypy.test.modpy::cpmodpysetup +PythonHandler cherrypy._cpmodpy::handler +PythonOption cherrypy.setup cherrypy.test.%(modulename)s::setup_server +PythonOption socket_host %(host)s +PythonDebug On +""" + +class ModPythonSupervisor(helper.Supervisor): + + using_apache = True + using_wsgi = False + template = None + + def __str__(self): + return "ModPython Server on %s:%s" % (self.host, self.port) + + def start(self, modulename): + mpconf = CONF_PATH + if not os.path.isabs(mpconf): + mpconf = os.path.join(curdir, mpconf) + + f = open(mpconf, 'wb') + try: + f.write(self.template % + {'port': self.port, 'modulename': modulename, + 'host': self.host}) + finally: + f.close() + + result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) + if result: + print(result) + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + read_process(APACHE_PATH, "-k stop") + + +loaded = False +def wsgisetup(req): + global loaded + if not loaded: + loaded = True + options = req.get_options() + + import cherrypy + cherrypy.config.update({ + "log.error_file": os.path.join(curdir, "test.log"), + "environment": "test_suite", + "server.socket_host": options['socket_host'], + }) + + modname = options['testmod'] + mod = __import__(modname, globals(), locals(), ['']) + mod.setup_server() + + cherrypy.server.unsubscribe() + cherrypy.engine.start() + from mod_python import apache + return apache.OK + + +def cpmodpysetup(req): + global loaded + if not loaded: + loaded = True + options = req.get_options() + + import cherrypy + cherrypy.config.update({ + "log.error_file": os.path.join(curdir, "test.log"), + "environment": "test_suite", + "server.socket_host": options['socket_host'], + }) + from mod_python import apache + return apache.OK + diff --git a/cherrypy/test/modwsgi.py b/cherrypy/test/modwsgi.py new file mode 100755 index 0000000..309a541 --- /dev/null +++ b/cherrypy/test/modwsgi.py @@ -0,0 +1,148 @@ +"""Wrapper for mod_wsgi, for use as a CherryPy HTTP server. + +To autostart modwsgi, the "apache" executable or script must be +on your system path, or you must override the global APACHE_PATH. +On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- +create a symlink to them if needed. + + +KNOWN BUGS +========== + +##1. Apache processes Range headers automatically; CherryPy's truncated +## output is then truncated again by Apache. See test_core.testRanges. +## This was worked around in http://www.cherrypy.org/changeset/1319. +2. Apache does not allow custom HTTP methods like CONNECT as per the spec. + See test_core.testHTTPMethods. +3. Max request header and body settings do not work with Apache. +##4. Apache replaces status "reason phrases" automatically. For example, +## CherryPy may set "304 Not modified" but Apache will write out +## "304 Not Modified" (capital "M"). +##5. Apache does not allow custom error codes as per the spec. +##6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the +## Request-URI too early. +7. mod_wsgi will not read request bodies which use the "chunked" + transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block + instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and + mod_python's requestobject.c). +8. When responding with 204 No Content, mod_wsgi adds a Content-Length + header for you. +9. When an error is raised, mod_wsgi has no facility for printing a + traceback as the response content (it's sent to the Apache log instead). +10. Startup and shutdown of Apache when running mod_wsgi seems slow. +""" + +import os +curdir = os.path.abspath(os.path.dirname(__file__)) +import re +import sys +import time + +import cherrypy +from cherrypy.test import helper, webtest + + +def read_process(cmd, args=""): + pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + try: + firstline = pipeout.readline() + if (re.search(r"(not recognized|No such file|not found)", firstline, + re.IGNORECASE)): + raise IOError('%s must be on your system path.' % cmd) + output = firstline + pipeout.read() + finally: + pipeout.close() + return output + + +if sys.platform == 'win32': + APACHE_PATH = "httpd" +else: + APACHE_PATH = "apache" + +CONF_PATH = "test_mw.conf" + +conf_modwsgi = r""" +# Apache2 server conf file for testing CherryPy with modpython_gateway. + +ServerName 127.0.0.1 +DocumentRoot "/" +Listen %(port)s + +AllowEncodedSlashes On +LoadModule rewrite_module modules/mod_rewrite.so +RewriteEngine on +RewriteMap escaping int:escape + +LoadModule log_config_module modules/mod_log_config.so +LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-agent}i\"" combined +CustomLog "%(curdir)s/apache.access.log" combined +ErrorLog "%(curdir)s/apache.error.log" +LogLevel debug + +LoadModule wsgi_module modules/mod_wsgi.so +LoadModule env_module modules/mod_env.so + +WSGIScriptAlias / "%(curdir)s/modwsgi.py" +SetEnv testmod %(testmod)s +""" + + +class ModWSGISupervisor(helper.Supervisor): + """Server Controller for ModWSGI and CherryPy.""" + + using_apache = True + using_wsgi = True + template=conf_modwsgi + + def __str__(self): + return "ModWSGI Server on %s:%s" % (self.host, self.port) + + def start(self, modulename): + mpconf = CONF_PATH + if not os.path.isabs(mpconf): + mpconf = os.path.join(curdir, mpconf) + + f = open(mpconf, 'wb') + try: + output = (self.template % + {'port': self.port, 'testmod': modulename, + 'curdir': curdir}) + f.write(output) + finally: + f.close() + + result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) + if result: + print(result) + + # Make a request so mod_wsgi starts up our app. + # If we don't, concurrent initial requests will 404. + cherrypy._cpserver.wait_for_occupied_port("127.0.0.1", self.port) + webtest.openURL('/ihopetheresnodefault', port=self.port) + time.sleep(1) + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + read_process(APACHE_PATH, "-k stop") + + +loaded = False +def application(environ, start_response): + import cherrypy + global loaded + if not loaded: + loaded = True + modname = "cherrypy.test." + environ['testmod'] + mod = __import__(modname, globals(), locals(), ['']) + mod.setup_server() + + cherrypy.config.update({ + "log.error_file": os.path.join(curdir, "test.error.log"), + "log.access_file": os.path.join(curdir, "test.access.log"), + "environment": "test_suite", + "engine.SIGHUP": None, + "engine.SIGTERM": None, + }) + return cherrypy.tree(environ, start_response) + diff --git a/cherrypy/test/sessiondemo.py b/cherrypy/test/sessiondemo.py new file mode 100755 index 0000000..342e5b5 --- /dev/null +++ b/cherrypy/test/sessiondemo.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +"""A session demonstration app.""" + +import calendar +from datetime import datetime +import sys +import cherrypy +from cherrypy.lib import sessions +from cherrypy._cpcompat import copyitems + + +page = """ +<html> +<head> +<style type='text/css'> +table { border-collapse: collapse; border: 1px solid #663333; } +th { text-align: right; background-color: #663333; color: white; padding: 0.5em; } +td { white-space: pre-wrap; font-family: monospace; padding: 0.5em; + border: 1px solid #663333; } +.warn { font-family: serif; color: #990000; } +</style> +<script type="text/javascript"> +<!-- +function twodigit(d) { return d < 10 ? "0" + d : d; } +function formattime(t) { + var month = t.getUTCMonth() + 1; + var day = t.getUTCDate(); + var year = t.getUTCFullYear(); + var hours = t.getUTCHours(); + var minutes = t.getUTCMinutes(); + return (year + "/" + twodigit(month) + "/" + twodigit(day) + " " + + hours + ":" + twodigit(minutes) + " UTC"); +} + +function interval(s) { + // Return the given interval (in seconds) as an English phrase + var seconds = s %% 60; + s = Math.floor(s / 60); + var minutes = s %% 60; + s = Math.floor(s / 60); + var hours = s %% 24; + var v = twodigit(hours) + ":" + twodigit(minutes) + ":" + twodigit(seconds); + var days = Math.floor(s / 24); + if (days != 0) v = days + ' days, ' + v; + return v; +} + +var fudge_seconds = 5; + +function init() { + // Set the content of the 'btime' cell. + var currentTime = new Date(); + var bunixtime = Math.floor(currentTime.getTime() / 1000); + + var v = formattime(currentTime); + v += " (Unix time: " + bunixtime + ")"; + + var diff = Math.abs(%(serverunixtime)s - bunixtime); + if (diff > fudge_seconds) v += "<p class='warn'>Browser and Server times disagree.</p>"; + + document.getElementById('btime').innerHTML = v; + + // Warn if response cookie expires is not close to one hour in the future. + // Yes, we want this to happen when wit hit the 'Expire' link, too. + var expires = Date.parse("%(expires)s") / 1000; + var onehour = (60 * 60); + if (Math.abs(expires - (bunixtime + onehour)) > fudge_seconds) { + diff = Math.floor(expires - bunixtime); + if (expires > (bunixtime + onehour)) { + var msg = "Response cookie 'expires' date is " + interval(diff) + " in the future."; + } else { + var msg = "Response cookie 'expires' date is " + interval(0 - diff) + " in the past."; + } + document.getElementById('respcookiewarn').innerHTML = msg; + } +} +//--> +</script> +</head> + +<body onload='init()'> +<h2>Session Demo</h2> +<p>Reload this page. The session ID should not change from one reload to the next</p> +<p><a href='../'>Index</a> | <a href='expire'>Expire</a> | <a href='regen'>Regenerate</a></p> +<table> + <tr><th>Session ID:</th><td>%(sessionid)s<p class='warn'>%(changemsg)s</p></td></tr> + <tr><th>Request Cookie</th><td>%(reqcookie)s</td></tr> + <tr><th>Response Cookie</th><td>%(respcookie)s<p id='respcookiewarn' class='warn'></p></td></tr> + <tr><th>Session Data</th><td>%(sessiondata)s</td></tr> + <tr><th>Server Time</th><td id='stime'>%(servertime)s (Unix time: %(serverunixtime)s)</td></tr> + <tr><th>Browser Time</th><td id='btime'> </td></tr> + <tr><th>Cherrypy Version:</th><td>%(cpversion)s</td></tr> + <tr><th>Python Version:</th><td>%(pyversion)s</td></tr> +</table> +</body></html> +""" + +class Root(object): + + def page(self): + changemsg = [] + if cherrypy.session.id != cherrypy.session.originalid: + if cherrypy.session.originalid is None: + changemsg.append('Created new session because no session id was given.') + if cherrypy.session.missing: + changemsg.append('Created new session due to missing (expired or malicious) session.') + if cherrypy.session.regenerated: + changemsg.append('Application generated a new session.') + + try: + expires = cherrypy.response.cookie['session_id']['expires'] + except KeyError: + expires = '' + + return page % { + 'sessionid': cherrypy.session.id, + 'changemsg': '<br>'.join(changemsg), + 'respcookie': cherrypy.response.cookie.output(), + 'reqcookie': cherrypy.request.cookie.output(), + 'sessiondata': copyitems(cherrypy.session), + 'servertime': datetime.utcnow().strftime("%Y/%m/%d %H:%M") + " UTC", + 'serverunixtime': calendar.timegm(datetime.utcnow().timetuple()), + 'cpversion': cherrypy.__version__, + 'pyversion': sys.version, + 'expires': expires, + } + + def index(self): + # Must modify data or the session will not be saved. + cherrypy.session['color'] = 'green' + return self.page() + index.exposed = True + + def expire(self): + sessions.expire() + return self.page() + expire.exposed = True + + def regen(self): + cherrypy.session.regenerate() + # Must modify data or the session will not be saved. + cherrypy.session['color'] = 'yellow' + return self.page() + regen.exposed = True + +if __name__ == '__main__': + cherrypy.config.update({ + #'environment': 'production', + 'log.screen': True, + 'tools.sessions.on': True, + }) + cherrypy.quickstart(Root()) + diff --git a/cherrypy/test/static/dirback.jpg b/cherrypy/test/static/dirback.jpg Binary files differnew file mode 100644 index 0000000..530e6d6 --- /dev/null +++ b/cherrypy/test/static/dirback.jpg diff --git a/cherrypy/test/static/index.html b/cherrypy/test/static/index.html new file mode 100644 index 0000000..b9f5f09 --- /dev/null +++ b/cherrypy/test/static/index.html @@ -0,0 +1 @@ +Hello, world
diff --git a/cherrypy/test/style.css b/cherrypy/test/style.css new file mode 100644 index 0000000..b266e93 --- /dev/null +++ b/cherrypy/test/style.css @@ -0,0 +1 @@ +Dummy stylesheet diff --git a/cherrypy/test/test.pem b/cherrypy/test/test.pem new file mode 100644 index 0000000..47a4704 --- /dev/null +++ b/cherrypy/test/test.pem @@ -0,0 +1,38 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQDBKo554mzIMY+AByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZ +R9L4WtImEew05FY3Izerfm3MN3+MC0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Kn +da+O6xldVSosu8Ev3z9VZ94iC/ZgKzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQAB +AoGAWOCF0ZrWxn3XMucWq2LNwPKqlvVGwbIwX3cDmX22zmnM4Fy6arXbYh4XlyCj +9+ofqRrxIFz5k/7tFriTmZ0xag5+Jdx+Kwg0/twiP7XCNKipFogwe1Hznw8OFAoT +enKBdj2+/n2o0Bvo/tDB59m9L/538d46JGQUmJlzMyqYikECQQDyoq+8CtMNvE18 +8VgHcR/KtApxWAjj4HpaHYL637ATjThetUZkW92mgDgowyplthusxdNqhHWyv7E8 +tWNdYErZAkEAy85ShTR0M5aWmrE7o0r0SpWInAkNBH9aXQRRARFYsdBtNfRu6I0i +0lvU9wiu3eF57FMEC86yViZ5UBnQfTu7vQJAVesj/Zt7pwaCDfdMa740OsxMUlyR +MVhhGx4OLpYdPJ8qUecxGQKq13XZ7R1HGyNEY4bd2X80Smq08UFuATfC6QJAH8UB +yBHtKz2GLIcELOg6PIYizW/7v3+6rlVF60yw7sb2vzpjL40QqIn4IKoR2DSVtOkb +8FtAIX3N21aq0VrGYQJBAIPiaEc2AZ8Bq2GC4F3wOz/BxJ/izvnkiotR12QK4fh5 +yjZMhTjWCas5zwHR5PDjlD88AWGDMsZ1PicD4348xJQ= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDxTCCAy6gAwIBAgIJAI18BD7eQxlGMA0GCSqGSIb3DQEBBAUAMIGeMQswCQYD +VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJU2FuIERpZWdv +MRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0MREwDwYDVQQLEwhkZXYtdGVzdDEW +MBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4GCSqGSIb3DQEJARYRcmVtaUBjaGVy +cnlweS5vcmcwHhcNMDYwOTA5MTkyMDIwWhcNMzQwMTI0MTkyMDIwWjCBnjELMAkG +A1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVNhbiBEaWVn +bzEZMBcGA1UEChMQQ2hlcnJ5UHkgUHJvamVjdDERMA8GA1UECxMIZGV2LXRlc3Qx +FjAUBgNVBAMTDUNoZXJyeVB5IFRlYW0xIDAeBgkqhkiG9w0BCQEWEXJlbWlAY2hl +cnJ5cHkub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDBKo554mzIMY+A +ByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZR9L4WtImEew05FY3Izerfm3MN3+M +C0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Knda+O6xldVSosu8Ev3z9VZ94iC/Zg +KzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQABo4IBBzCCAQMwHQYDVR0OBBYEFDIQ +2feb71tVZCWpU0qJ/Tw+wdtoMIHTBgNVHSMEgcswgciAFDIQ2feb71tVZCWpU0qJ +/Tw+wdtooYGkpIGhMIGeMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p +YTESMBAGA1UEBxMJU2FuIERpZWdvMRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0 +MREwDwYDVQQLEwhkZXYtdGVzdDEWMBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4G +CSqGSIb3DQEJARYRcmVtaUBjaGVycnlweS5vcmeCCQCNfAQ+3kMZRjAMBgNVHRME +BTADAQH/MA0GCSqGSIb3DQEBBAUAA4GBAL7AAQz7IePV48ZTAFHKr88ntPALsL5S +8vHCZPNMevNkLTj3DYUw2BcnENxMjm1kou2F2BkvheBPNZKIhc6z4hAml3ed1xa2 +D7w6e6OTcstdK/+KrPDDHeOP1dhMWNs2JE1bNlfF1LiXzYKSXpe88eCKjCXsCT/T +NluCaWQys3MS +-----END CERTIFICATE----- diff --git a/cherrypy/test/test_auth_basic.py b/cherrypy/test/test_auth_basic.py new file mode 100755 index 0000000..3a9781d --- /dev/null +++ b/cherrypy/test/test_auth_basic.py @@ -0,0 +1,79 @@ +# This file is part of CherryPy <http://www.cherrypy.org/> +# -*- coding: utf-8 -*- +# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 + +import cherrypy +from cherrypy._cpcompat import md5, ntob +from cherrypy.lib import auth_basic +from cherrypy.test import helper + + +class BasicAuthTest(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self): + return "This is public." + index.exposed = True + + class BasicProtected: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + class BasicProtected2: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + userpassdict = {'xuser' : 'xpassword'} + userhashdict = {'xuser' : md5(ntob('xpassword')).hexdigest()} + + def checkpasshash(realm, user, password): + p = userhashdict.get(user) + return p and p == md5(ntob(password)).hexdigest() or False + + conf = {'/basic': {'tools.auth_basic.on': True, + 'tools.auth_basic.realm': 'wonderland', + 'tools.auth_basic.checkpassword': auth_basic.checkpassword_dict(userpassdict)}, + '/basic2': {'tools.auth_basic.on': True, + 'tools.auth_basic.realm': 'wonderland', + 'tools.auth_basic.checkpassword': checkpasshash}, + } + + root = Root() + root.basic = BasicProtected() + root.basic2 = BasicProtected2() + cherrypy.tree.mount(root, config=conf) + setup_server = staticmethod(setup_server) + + def testPublic(self): + self.getPage("/") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.assertBody('This is public.') + + def testBasic(self): + self.getPage("/basic/") + self.assertStatus(401) + self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"') + + self.getPage('/basic/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')]) + self.assertStatus(401) + + self.getPage('/basic/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')]) + self.assertStatus('200 OK') + self.assertBody("Hello xuser, you've been authorized.") + + def testBasic2(self): + self.getPage("/basic2/") + self.assertStatus(401) + self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"') + + self.getPage('/basic2/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')]) + self.assertStatus(401) + + self.getPage('/basic2/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')]) + self.assertStatus('200 OK') + self.assertBody("Hello xuser, you've been authorized.") + diff --git a/cherrypy/test/test_auth_digest.py b/cherrypy/test/test_auth_digest.py new file mode 100755 index 0000000..1960fa8 --- /dev/null +++ b/cherrypy/test/test_auth_digest.py @@ -0,0 +1,115 @@ +# This file is part of CherryPy <http://www.cherrypy.org/> +# -*- coding: utf-8 -*- +# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 + + +import cherrypy +from cherrypy.lib import auth_digest + +from cherrypy.test import helper + +class DigestAuthTest(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self): + return "This is public." + index.exposed = True + + class DigestProtected: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + def fetch_users(): + return {'test': 'test'} + + + get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(fetch_users()) + conf = {'/digest': {'tools.auth_digest.on': True, + 'tools.auth_digest.realm': 'localhost', + 'tools.auth_digest.get_ha1': get_ha1, + 'tools.auth_digest.key': 'a565c27146791cfb', + 'tools.auth_digest.debug': 'True'}} + + root = Root() + root.digest = DigestProtected() + cherrypy.tree.mount(root, config=conf) + setup_server = staticmethod(setup_server) + + def testPublic(self): + self.getPage("/") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.assertBody('This is public.') + + def testDigest(self): + self.getPage("/digest/") + self.assertStatus(401) + + value = None + for k, v in self.headers: + if k.lower() == "www-authenticate": + if v.startswith("Digest"): + value = v + break + + if value is None: + self._handlewebError("Digest authentification scheme was not found") + + value = value[7:] + items = value.split(', ') + tokens = {} + for item in items: + key, value = item.split('=') + tokens[key.lower()] = value + + missing_msg = "%s is missing" + bad_value_msg = "'%s' was expecting '%s' but found '%s'" + nonce = None + if 'realm' not in tokens: + self._handlewebError(missing_msg % 'realm') + elif tokens['realm'] != '"localhost"': + self._handlewebError(bad_value_msg % ('realm', '"localhost"', tokens['realm'])) + if 'nonce' not in tokens: + self._handlewebError(missing_msg % 'nonce') + else: + nonce = tokens['nonce'].strip('"') + if 'algorithm' not in tokens: + self._handlewebError(missing_msg % 'algorithm') + elif tokens['algorithm'] != '"MD5"': + self._handlewebError(bad_value_msg % ('algorithm', '"MD5"', tokens['algorithm'])) + if 'qop' not in tokens: + self._handlewebError(missing_msg % 'qop') + elif tokens['qop'] != '"auth"': + self._handlewebError(bad_value_msg % ('qop', '"auth"', tokens['qop'])) + + get_ha1 = auth_digest.get_ha1_dict_plain({'test' : 'test'}) + + # Test user agent response with a wrong value for 'realm' + base_auth = 'Digest username="test", realm="wrong realm", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' + + auth_header = base_auth % (nonce, '11111111111111111111111111111111', '00000001') + auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET') + # calculate the response digest + ha1 = get_ha1(auth.realm, 'test') + response = auth.request_digest(ha1) + # send response with correct response digest, but wrong realm + auth_header = base_auth % (nonce, response, '00000001') + self.getPage('/digest/', [('Authorization', auth_header)]) + self.assertStatus(401) + + # Test that must pass + base_auth = 'Digest username="test", realm="localhost", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' + + auth_header = base_auth % (nonce, '11111111111111111111111111111111', '00000001') + auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET') + # calculate the response digest + ha1 = get_ha1('localhost', 'test') + response = auth.request_digest(ha1) + # send response with correct response digest + auth_header = base_auth % (nonce, response, '00000001') + self.getPage('/digest/', [('Authorization', auth_header)]) + self.assertStatus('200 OK') + self.assertBody("Hello test, you've been authorized.") + diff --git a/cherrypy/test/test_bus.py b/cherrypy/test/test_bus.py new file mode 100755 index 0000000..51c1022 --- /dev/null +++ b/cherrypy/test/test_bus.py @@ -0,0 +1,263 @@ +import threading +import time +import unittest + +import cherrypy +from cherrypy._cpcompat import get_daemon, set +from cherrypy.process import wspbus + + +msg = "Listener %d on channel %s: %s." + + +class PublishSubscribeTests(unittest.TestCase): + + def get_listener(self, channel, index): + def listener(arg=None): + self.responses.append(msg % (index, channel, arg)) + return listener + + def test_builtin_channels(self): + b = wspbus.Bus() + + self.responses, expected = [], [] + + for channel in b.listeners: + for index, priority in enumerate([100, 50, 0, 51]): + b.subscribe(channel, self.get_listener(channel, index), priority) + + for channel in b.listeners: + b.publish(channel) + expected.extend([msg % (i, channel, None) for i in (2, 1, 3, 0)]) + b.publish(channel, arg=79347) + expected.extend([msg % (i, channel, 79347) for i in (2, 1, 3, 0)]) + + self.assertEqual(self.responses, expected) + + def test_custom_channels(self): + b = wspbus.Bus() + + self.responses, expected = [], [] + + custom_listeners = ('hugh', 'louis', 'dewey') + for channel in custom_listeners: + for index, priority in enumerate([None, 10, 60, 40]): + b.subscribe(channel, self.get_listener(channel, index), priority) + + for channel in custom_listeners: + b.publish(channel, 'ah so') + expected.extend([msg % (i, channel, 'ah so') for i in (1, 3, 0, 2)]) + b.publish(channel) + expected.extend([msg % (i, channel, None) for i in (1, 3, 0, 2)]) + + self.assertEqual(self.responses, expected) + + def test_listener_errors(self): + b = wspbus.Bus() + + self.responses, expected = [], [] + channels = [c for c in b.listeners if c != 'log'] + + for channel in channels: + b.subscribe(channel, self.get_listener(channel, 1)) + # This will break since the lambda takes no args. + b.subscribe(channel, lambda: None, priority=20) + + for channel in channels: + self.assertRaises(wspbus.ChannelFailures, b.publish, channel, 123) + expected.append(msg % (1, channel, 123)) + + self.assertEqual(self.responses, expected) + + +class BusMethodTests(unittest.TestCase): + + def log(self, bus): + self._log_entries = [] + def logit(msg, level): + self._log_entries.append(msg) + bus.subscribe('log', logit) + + def assertLog(self, entries): + self.assertEqual(self._log_entries, entries) + + def get_listener(self, channel, index): + def listener(arg=None): + self.responses.append(msg % (index, channel, arg)) + return listener + + def test_start(self): + b = wspbus.Bus() + self.log(b) + + self.responses = [] + num = 3 + for index in range(num): + b.subscribe('start', self.get_listener('start', index)) + + b.start() + try: + # The start method MUST call all 'start' listeners. + self.assertEqual(set(self.responses), + set([msg % (i, 'start', None) for i in range(num)])) + # The start method MUST move the state to STARTED + # (or EXITING, if errors occur) + self.assertEqual(b.state, b.states.STARTED) + # The start method MUST log its states. + self.assertLog(['Bus STARTING', 'Bus STARTED']) + finally: + # Exit so the atexit handler doesn't complain. + b.exit() + + def test_stop(self): + b = wspbus.Bus() + self.log(b) + + self.responses = [] + num = 3 + for index in range(num): + b.subscribe('stop', self.get_listener('stop', index)) + + b.stop() + + # The stop method MUST call all 'stop' listeners. + self.assertEqual(set(self.responses), + set([msg % (i, 'stop', None) for i in range(num)])) + # The stop method MUST move the state to STOPPED + self.assertEqual(b.state, b.states.STOPPED) + # The stop method MUST log its states. + self.assertLog(['Bus STOPPING', 'Bus STOPPED']) + + def test_graceful(self): + b = wspbus.Bus() + self.log(b) + + self.responses = [] + num = 3 + for index in range(num): + b.subscribe('graceful', self.get_listener('graceful', index)) + + b.graceful() + + # The graceful method MUST call all 'graceful' listeners. + self.assertEqual(set(self.responses), + set([msg % (i, 'graceful', None) for i in range(num)])) + # The graceful method MUST log its states. + self.assertLog(['Bus graceful']) + + def test_exit(self): + b = wspbus.Bus() + self.log(b) + + self.responses = [] + num = 3 + for index in range(num): + b.subscribe('stop', self.get_listener('stop', index)) + b.subscribe('exit', self.get_listener('exit', index)) + + b.exit() + + # The exit method MUST call all 'stop' listeners, + # and then all 'exit' listeners. + self.assertEqual(set(self.responses), + set([msg % (i, 'stop', None) for i in range(num)] + + [msg % (i, 'exit', None) for i in range(num)])) + # The exit method MUST move the state to EXITING + self.assertEqual(b.state, b.states.EXITING) + # The exit method MUST log its states. + self.assertLog(['Bus STOPPING', 'Bus STOPPED', 'Bus EXITING', 'Bus EXITED']) + + def test_wait(self): + b = wspbus.Bus() + + def f(method): + time.sleep(0.2) + getattr(b, method)() + + for method, states in [('start', [b.states.STARTED]), + ('stop', [b.states.STOPPED]), + ('start', [b.states.STARTING, b.states.STARTED]), + ('exit', [b.states.EXITING]), + ]: + threading.Thread(target=f, args=(method,)).start() + b.wait(states) + + # The wait method MUST wait for the given state(s). + if b.state not in states: + self.fail("State %r not in %r" % (b.state, states)) + + def test_block(self): + b = wspbus.Bus() + self.log(b) + + def f(): + time.sleep(0.2) + b.exit() + def g(): + time.sleep(0.4) + threading.Thread(target=f).start() + threading.Thread(target=g).start() + threads = [t for t in threading.enumerate() if not get_daemon(t)] + self.assertEqual(len(threads), 3) + + b.block() + + # The block method MUST wait for the EXITING state. + self.assertEqual(b.state, b.states.EXITING) + # The block method MUST wait for ALL non-main, non-daemon threads to finish. + threads = [t for t in threading.enumerate() if not get_daemon(t)] + self.assertEqual(len(threads), 1) + # The last message will mention an indeterminable thread name; ignore it + self.assertEqual(self._log_entries[:-1], + ['Bus STOPPING', 'Bus STOPPED', + 'Bus EXITING', 'Bus EXITED', + 'Waiting for child threads to terminate...']) + + def test_start_with_callback(self): + b = wspbus.Bus() + self.log(b) + try: + events = [] + def f(*args, **kwargs): + events.append(("f", args, kwargs)) + def g(): + events.append("g") + b.subscribe("start", g) + b.start_with_callback(f, (1, 3, 5), {"foo": "bar"}) + # Give wait() time to run f() + time.sleep(0.2) + + # The callback method MUST wait for the STARTED state. + self.assertEqual(b.state, b.states.STARTED) + # The callback method MUST run after all start methods. + self.assertEqual(events, ["g", ("f", (1, 3, 5), {"foo": "bar"})]) + finally: + b.exit() + + def test_log(self): + b = wspbus.Bus() + self.log(b) + self.assertLog([]) + + # Try a normal message. + expected = [] + for msg in ["O mah darlin'"] * 3 + ["Clementiiiiiiiine"]: + b.log(msg) + expected.append(msg) + self.assertLog(expected) + + # Try an error message + try: + foo + except NameError: + b.log("You are lost and gone forever", traceback=True) + lastmsg = self._log_entries[-1] + if "Traceback" not in lastmsg or "NameError" not in lastmsg: + self.fail("Last log message %r did not contain " + "the expected traceback." % lastmsg) + else: + self.fail("NameError was not raised as expected.") + + +if __name__ == "__main__": + unittest.main() diff --git a/cherrypy/test/test_caching.py b/cherrypy/test/test_caching.py new file mode 100755 index 0000000..720a933 --- /dev/null +++ b/cherrypy/test/test_caching.py @@ -0,0 +1,329 @@ +import datetime +import gzip +from itertools import count +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +import sys +import threading +import time +import urllib + +import cherrypy +from cherrypy._cpcompat import next, ntob, quote, xrange +from cherrypy.lib import httputil + +gif_bytes = ntob('GIF89a\x01\x00\x01\x00\x82\x00\x01\x99"\x1e\x00\x00\x00\x00\x00' + '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + '\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x02\x03\x02\x08\t\x00;') + + + +from cherrypy.test import helper + +class CacheTest(helper.CPWebCase): + + def setup_server(): + + class Root: + + _cp_config = {'tools.caching.on': True} + + def __init__(self): + self.counter = 0 + self.control_counter = 0 + self.longlock = threading.Lock() + + def index(self): + self.counter += 1 + msg = "visit #%s" % self.counter + return msg + index.exposed = True + + def control(self): + self.control_counter += 1 + return "visit #%s" % self.control_counter + control.exposed = True + + def a_gif(self): + cherrypy.response.headers['Last-Modified'] = httputil.HTTPDate() + return gif_bytes + a_gif.exposed = True + + def long_process(self, seconds='1'): + try: + self.longlock.acquire() + time.sleep(float(seconds)) + finally: + self.longlock.release() + return 'success!' + long_process.exposed = True + + def clear_cache(self, path): + cherrypy._cache.store[cherrypy.request.base + path].clear() + clear_cache.exposed = True + + class VaryHeaderCachingServer(object): + + _cp_config = {'tools.caching.on': True, + 'tools.response_headers.on': True, + 'tools.response_headers.headers': [('Vary', 'Our-Varying-Header')], + } + + def __init__(self): + self.counter = count(1) + + def index(self): + return "visit #%s" % next(self.counter) + index.exposed = True + + class UnCached(object): + _cp_config = {'tools.expires.on': True, + 'tools.expires.secs': 60, + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.root': curdir, + } + + def force(self): + cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' + self._cp_config['tools.expires.force'] = True + self._cp_config['tools.expires.secs'] = 0 + return "being forceful" + force.exposed = True + force._cp_config = {'tools.expires.secs': 0} + + def dynamic(self): + cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' + cherrypy.response.headers['Cache-Control'] = 'private' + return "D-d-d-dynamic!" + dynamic.exposed = True + + def cacheable(self): + cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' + return "Hi, I'm cacheable." + cacheable.exposed = True + + def specific(self): + cherrypy.response.headers['Etag'] = 'need_this_to_make_me_cacheable' + return "I am being specific" + specific.exposed = True + specific._cp_config = {'tools.expires.secs': 86400} + + class Foo(object):pass + + def wrongtype(self): + cherrypy.response.headers['Etag'] = 'need_this_to_make_me_cacheable' + return "Woops" + wrongtype.exposed = True + wrongtype._cp_config = {'tools.expires.secs': Foo()} + + cherrypy.tree.mount(Root()) + cherrypy.tree.mount(UnCached(), "/expires") + cherrypy.tree.mount(VaryHeaderCachingServer(), "/varying_headers") + cherrypy.config.update({'tools.gzip.on': True}) + setup_server = staticmethod(setup_server) + + def testCaching(self): + elapsed = 0.0 + for trial in range(10): + self.getPage("/") + # The response should be the same every time, + # except for the Age response header. + self.assertBody('visit #1') + if trial != 0: + age = int(self.assertHeader("Age")) + self.assert_(age >= elapsed) + elapsed = age + + # POST, PUT, DELETE should not be cached. + self.getPage("/", method="POST") + self.assertBody('visit #2') + # Because gzip is turned on, the Vary header should always Vary for content-encoding + self.assertHeader('Vary', 'Accept-Encoding') + # The previous request should have invalidated the cache, + # so this request will recalc the response. + self.getPage("/", method="GET") + self.assertBody('visit #3') + # ...but this request should get the cached copy. + self.getPage("/", method="GET") + self.assertBody('visit #3') + self.getPage("/", method="DELETE") + self.assertBody('visit #4') + + # The previous request should have invalidated the cache, + # so this request will recalc the response. + self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')]) + self.assertHeader('Content-Encoding', 'gzip') + self.assertHeader('Vary') + self.assertEqual(cherrypy.lib.encoding.decompress(self.body), ntob("visit #5")) + + # Now check that a second request gets the gzip header and gzipped body + # This also tests a bug in 3.0 to 3.0.2 whereby the cached, gzipped + # response body was being gzipped a second time. + self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')]) + self.assertHeader('Content-Encoding', 'gzip') + self.assertEqual(cherrypy.lib.encoding.decompress(self.body), ntob("visit #5")) + + # Now check that a third request that doesn't accept gzip + # skips the cache (because the 'Vary' header denies it). + self.getPage("/", method="GET") + self.assertNoHeader('Content-Encoding') + self.assertBody('visit #6') + + def testVaryHeader(self): + self.getPage("/varying_headers/") + self.assertStatus("200 OK") + self.assertHeaderItemValue('Vary', 'Our-Varying-Header') + self.assertBody('visit #1') + + # Now check that different 'Vary'-fields don't evict each other. + # This test creates 2 requests with different 'Our-Varying-Header' + # and then tests if the first one still exists. + self.getPage("/varying_headers/", headers=[('Our-Varying-Header', 'request 2')]) + self.assertStatus("200 OK") + self.assertBody('visit #2') + + self.getPage("/varying_headers/", headers=[('Our-Varying-Header', 'request 2')]) + self.assertStatus("200 OK") + self.assertBody('visit #2') + + self.getPage("/varying_headers/") + self.assertStatus("200 OK") + self.assertBody('visit #1') + + def testExpiresTool(self): + # test setting an expires header + self.getPage("/expires/specific") + self.assertStatus("200 OK") + self.assertHeader("Expires") + + # test exceptions for bad time values + self.getPage("/expires/wrongtype") + self.assertStatus(500) + self.assertInBody("TypeError") + + # static content should not have "cache prevention" headers + self.getPage("/expires/index.html") + self.assertStatus("200 OK") + self.assertNoHeader("Pragma") + self.assertNoHeader("Cache-Control") + self.assertHeader("Expires") + + # dynamic content that sets indicators should not have + # "cache prevention" headers + self.getPage("/expires/cacheable") + self.assertStatus("200 OK") + self.assertNoHeader("Pragma") + self.assertNoHeader("Cache-Control") + self.assertHeader("Expires") + + self.getPage('/expires/dynamic') + self.assertBody("D-d-d-dynamic!") + # the Cache-Control header should be untouched + self.assertHeader("Cache-Control", "private") + self.assertHeader("Expires") + + # configure the tool to ignore indicators and replace existing headers + self.getPage("/expires/force") + self.assertStatus("200 OK") + # This also gives us a chance to test 0 expiry with no other headers + self.assertHeader("Pragma", "no-cache") + if cherrypy.server.protocol_version == "HTTP/1.1": + self.assertHeader("Cache-Control", "no-cache, must-revalidate") + self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") + + # static content should now have "cache prevention" headers + self.getPage("/expires/index.html") + self.assertStatus("200 OK") + self.assertHeader("Pragma", "no-cache") + if cherrypy.server.protocol_version == "HTTP/1.1": + self.assertHeader("Cache-Control", "no-cache, must-revalidate") + self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") + + # the cacheable handler should now have "cache prevention" headers + self.getPage("/expires/cacheable") + self.assertStatus("200 OK") + self.assertHeader("Pragma", "no-cache") + if cherrypy.server.protocol_version == "HTTP/1.1": + self.assertHeader("Cache-Control", "no-cache, must-revalidate") + self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") + + self.getPage('/expires/dynamic') + self.assertBody("D-d-d-dynamic!") + # dynamic sets Cache-Control to private but it should be + # overwritten here ... + self.assertHeader("Pragma", "no-cache") + if cherrypy.server.protocol_version == "HTTP/1.1": + self.assertHeader("Cache-Control", "no-cache, must-revalidate") + self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") + + def testLastModified(self): + self.getPage("/a.gif") + self.assertStatus(200) + self.assertBody(gif_bytes) + lm1 = self.assertHeader("Last-Modified") + + # this request should get the cached copy. + self.getPage("/a.gif") + self.assertStatus(200) + self.assertBody(gif_bytes) + self.assertHeader("Age") + lm2 = self.assertHeader("Last-Modified") + self.assertEqual(lm1, lm2) + + # this request should match the cached copy, but raise 304. + self.getPage("/a.gif", [('If-Modified-Since', lm1)]) + self.assertStatus(304) + self.assertNoHeader("Last-Modified") + if not getattr(cherrypy.server, "using_apache", False): + self.assertHeader("Age") + + def test_antistampede(self): + SECONDS = 4 + # We MUST make an initial synchronous request in order to create the + # AntiStampedeCache object, and populate its selecting_headers, + # before the actual stampede. + self.getPage("/long_process?seconds=%d" % SECONDS) + self.assertBody('success!') + self.getPage("/clear_cache?path=" + + quote('/long_process?seconds=%d' % SECONDS, safe='')) + self.assertStatus(200) + sys.stdout.write("prepped... ") + sys.stdout.flush() + + start = datetime.datetime.now() + def run(): + self.getPage("/long_process?seconds=%d" % SECONDS) + # The response should be the same every time + self.assertBody('success!') + ts = [threading.Thread(target=run) for i in xrange(100)] + for t in ts: + t.start() + for t in ts: + t.join() + self.assertEqualDates(start, datetime.datetime.now(), + # Allow a second for our thread/TCP overhead etc. + seconds=SECONDS + 1.1) + + def test_cache_control(self): + self.getPage("/control") + self.assertBody('visit #1') + self.getPage("/control") + self.assertBody('visit #1') + + self.getPage("/control", headers=[('Cache-Control', 'no-cache')]) + self.assertBody('visit #2') + self.getPage("/control") + self.assertBody('visit #2') + + self.getPage("/control", headers=[('Pragma', 'no-cache')]) + self.assertBody('visit #3') + self.getPage("/control") + self.assertBody('visit #3') + + time.sleep(1) + self.getPage("/control", headers=[('Cache-Control', 'max-age=0')]) + self.assertBody('visit #4') + self.getPage("/control") + self.assertBody('visit #4') + diff --git a/cherrypy/test/test_config.py b/cherrypy/test/test_config.py new file mode 100755 index 0000000..a0bd8ab --- /dev/null +++ b/cherrypy/test/test_config.py @@ -0,0 +1,249 @@ +"""Tests for the CherryPy configuration system.""" + +import os, sys +localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + +from cherrypy._cpcompat import ntob, StringIO +import unittest + +import cherrypy + +def setup_server(): + + class Root: + + _cp_config = {'foo': 'this', + 'bar': 'that'} + + def __init__(self): + cherrypy.config.namespaces['db'] = self.db_namespace + + def db_namespace(self, k, v): + if k == "scheme": + self.db = v + + # @cherrypy.expose(alias=('global_', 'xyz')) + def index(self, key): + return cherrypy.request.config.get(key, "None") + index = cherrypy.expose(index, alias=('global_', 'xyz')) + + def repr(self, key): + return repr(cherrypy.request.config.get(key, None)) + repr.exposed = True + + def dbscheme(self): + return self.db + dbscheme.exposed = True + + def plain(self, x): + return x + plain.exposed = True + plain._cp_config = {'request.body.attempt_charsets': ['utf-16']} + + favicon_ico = cherrypy.tools.staticfile.handler( + filename=os.path.join(localDir, '../favicon.ico')) + + class Foo: + + _cp_config = {'foo': 'this2', + 'baz': 'that2'} + + def index(self, key): + return cherrypy.request.config.get(key, "None") + index.exposed = True + nex = index + + def silly(self): + return 'Hello world' + silly.exposed = True + silly._cp_config = {'response.headers.X-silly': 'sillyval'} + + def bar(self, key): + return repr(cherrypy.request.config.get(key, None)) + bar.exposed = True + bar._cp_config = {'foo': 'this3', 'bax': 'this4'} + + class Another: + + def index(self, key): + return str(cherrypy.request.config.get(key, "None")) + index.exposed = True + + + def raw_namespace(key, value): + if key == 'input.map': + handler = cherrypy.request.handler + def wrapper(): + params = cherrypy.request.params + for name, coercer in list(value.items()): + try: + params[name] = coercer(params[name]) + except KeyError: + pass + return handler() + cherrypy.request.handler = wrapper + elif key == 'output': + handler = cherrypy.request.handler + def wrapper(): + # 'value' is a type (like int or str). + return value(handler()) + cherrypy.request.handler = wrapper + + class Raw: + + _cp_config = {'raw.output': repr} + + def incr(self, num): + return num + 1 + incr.exposed = True + incr._cp_config = {'raw.input.map': {'num': int}} + + ioconf = StringIO(""" +[/] +neg: -1234 +filename: os.path.join(sys.prefix, "hello.py") +thing1: cherrypy.lib.httputil.response_codes[404] +thing2: __import__('cherrypy.tutorial', globals(), locals(), ['']).thing2 +complex: 3+2j +ones: "11" +twos: "22" +stradd: %%(ones)s + %%(twos)s + "33" + +[/favicon.ico] +tools.staticfile.filename = %r +""" % os.path.join(localDir, 'static/dirback.jpg')) + + root = Root() + root.foo = Foo() + root.raw = Raw() + app = cherrypy.tree.mount(root, config=ioconf) + app.request_class.namespaces['raw'] = raw_namespace + + cherrypy.tree.mount(Another(), "/another") + cherrypy.config.update({'luxuryyacht': 'throatwobblermangrove', + 'db.scheme': r"sqlite///memory", + }) + + +# Client-side code # + +from cherrypy.test import helper + +class ConfigTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def testConfig(self): + tests = [ + ('/', 'nex', 'None'), + ('/', 'foo', 'this'), + ('/', 'bar', 'that'), + ('/xyz', 'foo', 'this'), + ('/foo/', 'foo', 'this2'), + ('/foo/', 'bar', 'that'), + ('/foo/', 'bax', 'None'), + ('/foo/bar', 'baz', "'that2'"), + ('/foo/nex', 'baz', 'that2'), + # If 'foo' == 'this', then the mount point '/another' leaks into '/'. + ('/another/','foo', 'None'), + ] + for path, key, expected in tests: + self.getPage(path + "?key=" + key) + self.assertBody(expected) + + expectedconf = { + # From CP defaults + 'tools.log_headers.on': False, + 'tools.log_tracebacks.on': True, + 'request.show_tracebacks': True, + 'log.screen': False, + 'environment': 'test_suite', + 'engine.autoreload_on': False, + # From global config + 'luxuryyacht': 'throatwobblermangrove', + # From Root._cp_config + 'bar': 'that', + # From Foo._cp_config + 'baz': 'that2', + # From Foo.bar._cp_config + 'foo': 'this3', + 'bax': 'this4', + } + for key, expected in expectedconf.items(): + self.getPage("/foo/bar?key=" + key) + self.assertBody(repr(expected)) + + def testUnrepr(self): + self.getPage("/repr?key=neg") + self.assertBody("-1234") + + self.getPage("/repr?key=filename") + self.assertBody(repr(os.path.join(sys.prefix, "hello.py"))) + + self.getPage("/repr?key=thing1") + self.assertBody(repr(cherrypy.lib.httputil.response_codes[404])) + + if not getattr(cherrypy.server, "using_apache", False): + # The object ID's won't match up when using Apache, since the + # server and client are running in different processes. + self.getPage("/repr?key=thing2") + from cherrypy.tutorial import thing2 + self.assertBody(repr(thing2)) + + self.getPage("/repr?key=complex") + self.assertBody("(3+2j)") + + self.getPage("/repr?key=stradd") + self.assertBody(repr("112233")) + + def testRespNamespaces(self): + self.getPage("/foo/silly") + self.assertHeader('X-silly', 'sillyval') + self.assertBody('Hello world') + + def testCustomNamespaces(self): + self.getPage("/raw/incr?num=12") + self.assertBody("13") + + self.getPage("/dbscheme") + self.assertBody(r"sqlite///memory") + + def testHandlerToolConfigOverride(self): + # Assert that config overrides tool constructor args. Above, we set + # the favicon in the page handler to be '../favicon.ico', + # but then overrode it in config to be './static/dirback.jpg'. + self.getPage("/favicon.ico") + self.assertBody(open(os.path.join(localDir, "static/dirback.jpg"), + "rb").read()) + + def test_request_body_namespace(self): + self.getPage("/plain", method='POST', headers=[ + ('Content-Type', 'application/x-www-form-urlencoded'), + ('Content-Length', '13')], + body=ntob('\xff\xfex\x00=\xff\xfea\x00b\x00c\x00')) + self.assertBody("abc") + + +class VariableSubstitutionTests(unittest.TestCase): + setup_server = staticmethod(setup_server) + + def test_config(self): + from textwrap import dedent + + # variable substitution with [DEFAULT] + conf = dedent(""" + [DEFAULT] + dir = "/some/dir" + my.dir = %(dir)s + "/sub" + + [my] + my.dir = %(dir)s + "/my/dir" + my.dir2 = %(my.dir)s + '/dir2' + + """) + + fp = StringIO(conf) + + cherrypy.config.update(fp) + self.assertEqual(cherrypy.config["my"]["my.dir"], "/some/dir/my/dir") + self.assertEqual(cherrypy.config["my"]["my.dir2"], "/some/dir/my/dir/dir2") + diff --git a/cherrypy/test/test_config_server.py b/cherrypy/test/test_config_server.py new file mode 100755 index 0000000..0b9718d --- /dev/null +++ b/cherrypy/test/test_config_server.py @@ -0,0 +1,121 @@ +"""Tests for the CherryPy configuration system.""" + +import os, sys +localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +import socket +import time + +import cherrypy + + +# Client-side code # + +from cherrypy.test import helper + +class ServerConfigTests(helper.CPWebCase): + + def setup_server(): + + class Root: + def index(self): + return cherrypy.request.wsgi_environ['SERVER_PORT'] + index.exposed = True + + def upload(self, file): + return "Size: %s" % len(file.file.read()) + upload.exposed = True + + def tinyupload(self): + return cherrypy.request.body.read() + tinyupload.exposed = True + tinyupload._cp_config = {'request.body.maxbytes': 100} + + cherrypy.tree.mount(Root()) + + cherrypy.config.update({ + 'server.socket_host': '0.0.0.0', + 'server.socket_port': 9876, + 'server.max_request_body_size': 200, + 'server.max_request_header_size': 500, + 'server.socket_timeout': 0.5, + + # Test explicit server.instance + 'server.2.instance': 'cherrypy._cpwsgi_server.CPWSGIServer', + 'server.2.socket_port': 9877, + + # Test non-numeric <servername> + # Also test default server.instance = builtin server + 'server.yetanother.socket_port': 9878, + }) + setup_server = staticmethod(setup_server) + + PORT = 9876 + + def testBasicConfig(self): + self.getPage("/") + self.assertBody(str(self.PORT)) + + def testAdditionalServers(self): + if self.scheme == 'https': + return self.skip("not available under ssl") + self.PORT = 9877 + self.getPage("/") + self.assertBody(str(self.PORT)) + self.PORT = 9878 + self.getPage("/") + self.assertBody(str(self.PORT)) + + def testMaxRequestSizePerHandler(self): + if getattr(cherrypy.server, "using_apache", False): + return self.skip("skipped due to known Apache differences... ") + + self.getPage('/tinyupload', method="POST", + headers=[('Content-Type', 'text/plain'), + ('Content-Length', '100')], + body="x" * 100) + self.assertStatus(200) + self.assertBody("x" * 100) + + self.getPage('/tinyupload', method="POST", + headers=[('Content-Type', 'text/plain'), + ('Content-Length', '101')], + body="x" * 101) + self.assertStatus(413) + + def testMaxRequestSize(self): + if getattr(cherrypy.server, "using_apache", False): + return self.skip("skipped due to known Apache differences... ") + + for size in (500, 5000, 50000): + self.getPage("/", headers=[('From', "x" * 500)]) + self.assertStatus(413) + + # Test for http://www.cherrypy.org/ticket/421 + # (Incorrect border condition in readline of SizeCheckWrapper). + # This hangs in rev 891 and earlier. + lines256 = "x" * 248 + self.getPage("/", + headers=[('Host', '%s:%s' % (self.HOST, self.PORT)), + ('From', lines256)]) + + # Test upload + body = '\r\n'.join([ + '--x', + 'Content-Disposition: form-data; name="file"; filename="hello.txt"', + 'Content-Type: text/plain', + '', + '%s', + '--x--']) + partlen = 200 - len(body) + b = body % ("x" * partlen) + h = [("Content-type", "multipart/form-data; boundary=x"), + ("Content-Length", "%s" % len(b))] + self.getPage('/upload', h, "POST", b) + self.assertBody('Size: %d' % partlen) + + b = body % ("x" * 200) + h = [("Content-type", "multipart/form-data; boundary=x"), + ("Content-Length", "%s" % len(b))] + self.getPage('/upload', h, "POST", b) + self.assertStatus(413) + diff --git a/cherrypy/test/test_conn.py b/cherrypy/test/test_conn.py new file mode 100755 index 0000000..1346f59 --- /dev/null +++ b/cherrypy/test/test_conn.py @@ -0,0 +1,734 @@ +"""Tests for TCP connection handling, including proper and timely close.""" + +import socket +import sys +import time +timeout = 1 + + +import cherrypy +from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, NotConnected, BadStatusLine +from cherrypy._cpcompat import ntob, urlopen, unicodestr +from cherrypy.test import webtest +from cherrypy import _cperror + + +pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN' + +def setup_server(): + + def raise500(): + raise cherrypy.HTTPError(500) + + class Root: + + def index(self): + return pov + index.exposed = True + page1 = index + page2 = index + page3 = index + + def hello(self): + return "Hello, world!" + hello.exposed = True + + def timeout(self, t): + return str(cherrypy.server.httpserver.timeout) + timeout.exposed = True + + def stream(self, set_cl=False): + if set_cl: + cherrypy.response.headers['Content-Length'] = 10 + + def content(): + for x in range(10): + yield str(x) + + return content() + stream.exposed = True + stream._cp_config = {'response.stream': True} + + def error(self, code=500): + raise cherrypy.HTTPError(code) + error.exposed = True + + def upload(self): + if not cherrypy.request.method == 'POST': + raise AssertionError("'POST' != request.method %r" % + cherrypy.request.method) + return "thanks for '%s'" % cherrypy.request.body.read() + upload.exposed = True + + def custom(self, response_code): + cherrypy.response.status = response_code + return "Code = %s" % response_code + custom.exposed = True + + def err_before_read(self): + return "ok" + err_before_read.exposed = True + err_before_read._cp_config = {'hooks.on_start_resource': raise500} + + def one_megabyte_of_a(self): + return ["a" * 1024] * 1024 + one_megabyte_of_a.exposed = True + + def custom_cl(self, body, cl): + cherrypy.response.headers['Content-Length'] = cl + if not isinstance(body, list): + body = [body] + newbody = [] + for chunk in body: + if isinstance(chunk, unicodestr): + chunk = chunk.encode('ISO-8859-1') + newbody.append(chunk) + return newbody + custom_cl.exposed = True + # Turn off the encoding tool so it doens't collapse + # our response body and reclaculate the Content-Length. + custom_cl._cp_config = {'tools.encode.on': False} + + cherrypy.tree.mount(Root()) + cherrypy.config.update({ + 'server.max_request_body_size': 1001, + 'server.socket_timeout': timeout, + }) + + +from cherrypy.test import helper + +class ConnectionCloseTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_HTTP11(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + self.persistent = True + + # Make the first request and assert there's no "Connection: close". + self.getPage("/") + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertNoHeader("Connection") + + # Make another request on the same connection. + self.getPage("/page1") + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertNoHeader("Connection") + + # Test client-side close. + self.getPage("/page2", headers=[("Connection", "close")]) + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertHeader("Connection", "close") + + # Make another request on the same connection, which should error. + self.assertRaises(NotConnected, self.getPage, "/") + + def test_Streaming_no_len(self): + self._streaming(set_cl=False) + + def test_Streaming_with_len(self): + self._streaming(set_cl=True) + + def _streaming(self, set_cl): + if cherrypy.server.protocol_version == "HTTP/1.1": + self.PROTOCOL = "HTTP/1.1" + + self.persistent = True + + # Make the first request and assert there's no "Connection: close". + self.getPage("/") + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertNoHeader("Connection") + + # Make another, streamed request on the same connection. + if set_cl: + # When a Content-Length is provided, the content should stream + # without closing the connection. + self.getPage("/stream?set_cl=Yes") + self.assertHeader("Content-Length") + self.assertNoHeader("Connection", "close") + self.assertNoHeader("Transfer-Encoding") + + self.assertStatus('200 OK') + self.assertBody('0123456789') + else: + # When no Content-Length response header is provided, + # streamed output will either close the connection, or use + # chunked encoding, to determine transfer-length. + self.getPage("/stream") + self.assertNoHeader("Content-Length") + self.assertStatus('200 OK') + self.assertBody('0123456789') + + chunked_response = False + for k, v in self.headers: + if k.lower() == "transfer-encoding": + if str(v) == "chunked": + chunked_response = True + + if chunked_response: + self.assertNoHeader("Connection", "close") + else: + self.assertHeader("Connection", "close") + + # Make another request on the same connection, which should error. + self.assertRaises(NotConnected, self.getPage, "/") + + # Try HEAD. See http://www.cherrypy.org/ticket/864. + self.getPage("/stream", method='HEAD') + self.assertStatus('200 OK') + self.assertBody('') + self.assertNoHeader("Transfer-Encoding") + else: + self.PROTOCOL = "HTTP/1.0" + + self.persistent = True + + # Make the first request and assert Keep-Alive. + self.getPage("/", headers=[("Connection", "Keep-Alive")]) + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertHeader("Connection", "Keep-Alive") + + # Make another, streamed request on the same connection. + if set_cl: + # When a Content-Length is provided, the content should + # stream without closing the connection. + self.getPage("/stream?set_cl=Yes", + headers=[("Connection", "Keep-Alive")]) + self.assertHeader("Content-Length") + self.assertHeader("Connection", "Keep-Alive") + self.assertNoHeader("Transfer-Encoding") + self.assertStatus('200 OK') + self.assertBody('0123456789') + else: + # When a Content-Length is not provided, + # the server should close the connection. + self.getPage("/stream", headers=[("Connection", "Keep-Alive")]) + self.assertStatus('200 OK') + self.assertBody('0123456789') + + self.assertNoHeader("Content-Length") + self.assertNoHeader("Connection", "Keep-Alive") + self.assertNoHeader("Transfer-Encoding") + + # Make another request on the same connection, which should error. + self.assertRaises(NotConnected, self.getPage, "/") + + def test_HTTP10_KeepAlive(self): + self.PROTOCOL = "HTTP/1.0" + if self.scheme == "https": + self.HTTP_CONN = HTTPSConnection + else: + self.HTTP_CONN = HTTPConnection + + # Test a normal HTTP/1.0 request. + self.getPage("/page2") + self.assertStatus('200 OK') + self.assertBody(pov) + # Apache, for example, may emit a Connection header even for HTTP/1.0 +## self.assertNoHeader("Connection") + + # Test a keep-alive HTTP/1.0 request. + self.persistent = True + + self.getPage("/page3", headers=[("Connection", "Keep-Alive")]) + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertHeader("Connection", "Keep-Alive") + + # Remove the keep-alive header again. + self.getPage("/page3") + self.assertStatus('200 OK') + self.assertBody(pov) + # Apache, for example, may emit a Connection header even for HTTP/1.0 +## self.assertNoHeader("Connection") + + +class PipelineTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_HTTP11_Timeout(self): + # If we timeout without sending any data, + # the server will close the conn with a 408. + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Connect but send nothing. + self.persistent = True + conn = self.HTTP_CONN + conn.auto_open = False + conn.connect() + + # Wait for our socket timeout + time.sleep(timeout * 2) + + # The request should have returned 408 already. + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 408) + conn.close() + + # Connect but send half the headers only. + self.persistent = True + conn = self.HTTP_CONN + conn.auto_open = False + conn.connect() + conn.send(ntob('GET /hello HTTP/1.1')) + conn.send(("Host: %s" % self.HOST).encode('ascii')) + + # Wait for our socket timeout + time.sleep(timeout * 2) + + # The conn should have already sent 408. + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 408) + conn.close() + + def test_HTTP11_Timeout_after_request(self): + # If we timeout after at least one request has succeeded, + # the server will close the conn without 408. + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Make an initial request + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/timeout?t=%s" % timeout, skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + self.body = response.read() + self.assertBody(str(timeout)) + + # Make a second request on the same socket + conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(ntob("Host: %s" % self.HOST, 'ascii')) + conn._send_output() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + self.body = response.read() + self.assertBody("Hello, world!") + + # Wait for our socket timeout + time.sleep(timeout * 2) + + # Make another request on the same socket, which should error + conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(ntob("Host: %s" % self.HOST, 'ascii')) + conn._send_output() + response = conn.response_class(conn.sock, method="GET") + try: + response.begin() + except: + if not isinstance(sys.exc_info()[1], + (socket.error, BadStatusLine)): + self.fail("Writing to timed out socket didn't fail" + " as it should have: %s" % sys.exc_info()[1]) + else: + if response.status != 408: + self.fail("Writing to timed out socket didn't fail" + " as it should have: %s" % + response.read()) + + conn.close() + + # Make another request on a new socket, which should work + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + self.body = response.read() + self.assertBody(pov) + + + # Make another request on the same socket, + # but timeout on the headers + conn.send(ntob('GET /hello HTTP/1.1')) + # Wait for our socket timeout + time.sleep(timeout * 2) + response = conn.response_class(conn.sock, method="GET") + try: + response.begin() + except: + if not isinstance(sys.exc_info()[1], + (socket.error, BadStatusLine)): + self.fail("Writing to timed out socket didn't fail" + " as it should have: %s" % sys.exc_info()[1]) + else: + self.fail("Writing to timed out socket didn't fail" + " as it should have: %s" % + response.read()) + + conn.close() + + # Retry the request on a new connection, which should work + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + self.body = response.read() + self.assertBody(pov) + conn.close() + + def test_HTTP11_pipelining(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Test pipelining. httplib doesn't support this directly. + self.persistent = True + conn = self.HTTP_CONN + + # Put request 1 + conn.putrequest("GET", "/hello", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + + for trial in range(5): + # Put next request + conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(ntob("Host: %s" % self.HOST, 'ascii')) + conn._send_output() + + # Retrieve previous response + response = conn.response_class(conn.sock, method="GET") + response.begin() + body = response.read(13) + self.assertEqual(response.status, 200) + self.assertEqual(body, ntob("Hello, world!")) + + # Retrieve final response + response = conn.response_class(conn.sock, method="GET") + response.begin() + body = response.read() + self.assertEqual(response.status, 200) + self.assertEqual(body, ntob("Hello, world!")) + + conn.close() + + def test_100_Continue(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + self.persistent = True + conn = self.HTTP_CONN + + # Try a page without an Expect request header first. + # Note that httplib's response.begin automatically ignores + # 100 Continue responses, so we must manually check for it. + conn.putrequest("POST", "/upload", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Content-Type", "text/plain") + conn.putheader("Content-Length", "4") + conn.endheaders() + conn.send(ntob("d'oh")) + response = conn.response_class(conn.sock, method="POST") + version, status, reason = response._read_status() + self.assertNotEqual(status, 100) + conn.close() + + # Now try a page with an Expect header... + conn.connect() + conn.putrequest("POST", "/upload", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Content-Type", "text/plain") + conn.putheader("Content-Length", "17") + conn.putheader("Expect", "100-continue") + conn.endheaders() + response = conn.response_class(conn.sock, method="POST") + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + self.assertEqual(status, 100) + while True: + line = response.fp.readline().strip() + if line: + self.fail("100 Continue should not output any headers. Got %r" % line) + else: + break + + # ...send the body + body = ntob("I am a small file") + conn.send(body) + + # ...get the final response + response.begin() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(200) + self.assertBody("thanks for '%s'" % body) + conn.close() + + +class ConnectionTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_readall_or_close(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + if self.scheme == "https": + self.HTTP_CONN = HTTPSConnection + else: + self.HTTP_CONN = HTTPConnection + + # Test a max of 0 (the default) and then reset to what it was above. + old_max = cherrypy.server.max_request_body_size + for new_max in (0, old_max): + cherrypy.server.max_request_body_size = new_max + + self.persistent = True + conn = self.HTTP_CONN + + # Get a POST page with an error + conn.putrequest("POST", "/err_before_read", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Content-Type", "text/plain") + conn.putheader("Content-Length", "1000") + conn.putheader("Expect", "100-continue") + conn.endheaders() + response = conn.response_class(conn.sock, method="POST") + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + self.assertEqual(status, 100) + while True: + skip = response.fp.readline().strip() + if not skip: + break + + # ...send the body + conn.send(ntob("x" * 1000)) + + # ...get the final response + response.begin() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(500) + + # Now try a working page with an Expect header... + conn._output(ntob('POST /upload HTTP/1.1')) + conn._output(ntob("Host: %s" % self.HOST, 'ascii')) + conn._output(ntob("Content-Type: text/plain")) + conn._output(ntob("Content-Length: 17")) + conn._output(ntob("Expect: 100-continue")) + conn._send_output() + response = conn.response_class(conn.sock, method="POST") + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + self.assertEqual(status, 100) + while True: + skip = response.fp.readline().strip() + if not skip: + break + + # ...send the body + body = ntob("I am a small file") + conn.send(body) + + # ...get the final response + response.begin() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(200) + self.assertBody("thanks for '%s'" % body) + conn.close() + + def test_No_Message_Body(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Set our HTTP_CONN to an instance so it persists between requests. + self.persistent = True + + # Make the first request and assert there's no "Connection: close". + self.getPage("/") + self.assertStatus('200 OK') + self.assertBody(pov) + self.assertNoHeader("Connection") + + # Make a 204 request on the same connection. + self.getPage("/custom/204") + self.assertStatus(204) + self.assertNoHeader("Content-Length") + self.assertBody("") + self.assertNoHeader("Connection") + + # Make a 304 request on the same connection. + self.getPage("/custom/304") + self.assertStatus(304) + self.assertNoHeader("Content-Length") + self.assertBody("") + self.assertNoHeader("Connection") + + def test_Chunked_Encoding(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + if (hasattr(self, 'harness') and + "modpython" in self.harness.__class__.__name__.lower()): + # mod_python forbids chunked encoding + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Set our HTTP_CONN to an instance so it persists between requests. + self.persistent = True + conn = self.HTTP_CONN + + # Try a normal chunked request (with extensions) + body = ntob("8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n" + "Content-Type: application/json\r\n" + "\r\n") + conn.putrequest("POST", "/upload", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Transfer-Encoding", "chunked") + conn.putheader("Trailer", "Content-Type") + # Note that this is somewhat malformed: + # we shouldn't be sending Content-Length. + # RFC 2616 says the server should ignore it. + conn.putheader("Content-Length", "3") + conn.endheaders() + conn.send(body) + response = conn.getresponse() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus('200 OK') + self.assertBody("thanks for '%s'" % ntob('xx\r\nxxxxyyyyy')) + + # Try a chunked request that exceeds server.max_request_body_size. + # Note that the delimiters and trailer are included. + body = ntob("3e3\r\n" + ("x" * 995) + "\r\n0\r\n\r\n") + conn.putrequest("POST", "/upload", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Transfer-Encoding", "chunked") + conn.putheader("Content-Type", "text/plain") + # Chunked requests don't need a content-length +## conn.putheader("Content-Length", len(body)) + conn.endheaders() + conn.send(body) + response = conn.getresponse() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(413) + conn.close() + + def test_Content_Length_in(self): + # Try a non-chunked request where Content-Length exceeds + # server.max_request_body_size. Assert error before body send. + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("POST", "/upload", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader("Content-Type", "text/plain") + conn.putheader("Content-Length", "9999") + conn.endheaders() + response = conn.getresponse() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(413) + self.assertBody("The entity sent with the request exceeds " + "the maximum allowed bytes.") + conn.close() + + def test_Content_Length_out_preheaders(self): + # Try a non-chunked response where Content-Length is less than + # the actual bytes in the response body. + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/custom_cl?body=I+have+too+many+bytes&cl=5", + skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.getresponse() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(500) + self.assertBody( + "The requested resource returned more bytes than the " + "declared Content-Length.") + conn.close() + + def test_Content_Length_out_postheaders(self): + # Try a non-chunked response where Content-Length is less than + # the actual bytes in the response body. + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/custom_cl?body=I+too&body=+have+too+many&cl=5", + skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.getresponse() + self.status, self.headers, self.body = webtest.shb(response) + self.assertStatus(200) + self.assertBody("I too") + conn.close() + + def test_598(self): + remote_data_conn = urlopen('%s://%s:%s/one_megabyte_of_a/' % + (self.scheme, self.HOST, self.PORT,)) + buf = remote_data_conn.read(512) + time.sleep(timeout * 0.6) + remaining = (1024 * 1024) - 512 + while remaining: + data = remote_data_conn.read(remaining) + if not data: + break + else: + buf += data + remaining -= len(data) + + self.assertEqual(len(buf), 1024 * 1024) + self.assertEqual(buf, ntob("a" * 1024 * 1024)) + self.assertEqual(remaining, 0) + remote_data_conn.close() + + +class BadRequestTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_No_CRLF(self): + self.persistent = True + + conn = self.HTTP_CONN + conn.send(ntob('GET /hello HTTP/1.1\n\n')) + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.body = response.read() + self.assertBody("HTTP requires CRLF terminators") + conn.close() + + conn.connect() + conn.send(ntob('GET /hello HTTP/1.1\r\n\n')) + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.body = response.read() + self.assertBody("HTTP requires CRLF terminators") + conn.close() + diff --git a/cherrypy/test/test_core.py b/cherrypy/test/test_core.py new file mode 100755 index 0000000..09544e3 --- /dev/null +++ b/cherrypy/test/test_core.py @@ -0,0 +1,617 @@ +"""Basic tests for the CherryPy core: request handling.""" + +import os +localDir = os.path.dirname(__file__) +import sys +import types + +import cherrypy +from cherrypy._cpcompat import IncompleteRead, itervalues, ntob +from cherrypy import _cptools, tools +from cherrypy.lib import httputil, static + + +favicon_path = os.path.join(os.getcwd(), localDir, "../favicon.ico") + +# Client-side code # + +from cherrypy.test import helper + +class CoreRequestHandlingTest(helper.CPWebCase): + + def setup_server(): + class Root: + + def index(self): + return "hello" + index.exposed = True + + favicon_ico = tools.staticfile.handler(filename=favicon_path) + + def defct(self, newct): + newct = "text/%s" % newct + cherrypy.config.update({'tools.response_headers.on': True, + 'tools.response_headers.headers': + [('Content-Type', newct)]}) + defct.exposed = True + + def baseurl(self, path_info, relative=None): + return cherrypy.url(path_info, relative=bool(relative)) + baseurl.exposed = True + + root = Root() + + if sys.version_info >= (2, 5): + from cherrypy.test._test_decorators import ExposeExamples + root.expose_dec = ExposeExamples() + + + class TestType(type): + """Metaclass which automatically exposes all functions in each subclass, + and adds an instance of the subclass as an attribute of root. + """ + def __init__(cls, name, bases, dct): + type.__init__(cls, name, bases, dct) + for value in itervalues(dct): + if isinstance(value, types.FunctionType): + value.exposed = True + setattr(root, name.lower(), cls()) + class Test(object): + __metaclass__ = TestType + + + class URL(Test): + + _cp_config = {'tools.trailing_slash.on': False} + + def index(self, path_info, relative=None): + if relative != 'server': + relative = bool(relative) + return cherrypy.url(path_info, relative=relative) + + def leaf(self, path_info, relative=None): + if relative != 'server': + relative = bool(relative) + return cherrypy.url(path_info, relative=relative) + + + class Status(Test): + + def index(self): + return "normal" + + def blank(self): + cherrypy.response.status = "" + + # According to RFC 2616, new status codes are OK as long as they + # are between 100 and 599. + + # Here is an illegal code... + def illegal(self): + cherrypy.response.status = 781 + return "oops" + + # ...and here is an unknown but legal code. + def unknown(self): + cherrypy.response.status = "431 My custom error" + return "funky" + + # Non-numeric code + def bad(self): + cherrypy.response.status = "error" + return "bad news" + + + class Redirect(Test): + + class Error: + _cp_config = {"tools.err_redirect.on": True, + "tools.err_redirect.url": "/errpage", + "tools.err_redirect.internal": False, + } + + def index(self): + raise NameError("redirect_test") + index.exposed = True + error = Error() + + def index(self): + return "child" + + def custom(self, url, code): + raise cherrypy.HTTPRedirect(url, code) + + def by_code(self, code): + raise cherrypy.HTTPRedirect("somewhere%20else", code) + by_code._cp_config = {'tools.trailing_slash.extra': True} + + def nomodify(self): + raise cherrypy.HTTPRedirect("", 304) + + def proxy(self): + raise cherrypy.HTTPRedirect("proxy", 305) + + def stringify(self): + return str(cherrypy.HTTPRedirect("/")) + + def fragment(self, frag): + raise cherrypy.HTTPRedirect("/some/url#%s" % frag) + + def login_redir(): + if not getattr(cherrypy.request, "login", None): + raise cherrypy.InternalRedirect("/internalredirect/login") + tools.login_redir = _cptools.Tool('before_handler', login_redir) + + def redir_custom(): + raise cherrypy.InternalRedirect("/internalredirect/custom_err") + + class InternalRedirect(Test): + + def index(self): + raise cherrypy.InternalRedirect("/") + + def choke(self): + return 3 / 0 + choke.exposed = True + choke._cp_config = {'hooks.before_error_response': redir_custom} + + def relative(self, a, b): + raise cherrypy.InternalRedirect("cousin?t=6") + + def cousin(self, t): + assert cherrypy.request.prev.closed + return cherrypy.request.prev.query_string + + def petshop(self, user_id): + if user_id == "parrot": + # Trade it for a slug when redirecting + raise cherrypy.InternalRedirect('/image/getImagesByUser?user_id=slug') + elif user_id == "terrier": + # Trade it for a fish when redirecting + raise cherrypy.InternalRedirect('/image/getImagesByUser?user_id=fish') + else: + # This should pass the user_id through to getImagesByUser + raise cherrypy.InternalRedirect( + '/image/getImagesByUser?user_id=%s' % str(user_id)) + + # We support Python 2.3, but the @-deco syntax would look like this: + # @tools.login_redir() + def secure(self): + return "Welcome!" + secure = tools.login_redir()(secure) + # Since calling the tool returns the same function you pass in, + # you could skip binding the return value, and just write: + # tools.login_redir()(secure) + + def login(self): + return "Please log in" + + def custom_err(self): + return "Something went horribly wrong." + + def early_ir(self, arg): + return "whatever" + early_ir._cp_config = {'hooks.before_request_body': redir_custom} + + + class Image(Test): + + def getImagesByUser(self, user_id): + return "0 images for %s" % user_id + + + class Flatten(Test): + + def as_string(self): + return "content" + + def as_list(self): + return ["con", "tent"] + + def as_yield(self): + yield ntob("content") + + def as_dblyield(self): + yield self.as_yield() + as_dblyield._cp_config = {'tools.flatten.on': True} + + def as_refyield(self): + for chunk in self.as_yield(): + yield chunk + + + class Ranges(Test): + + def get_ranges(self, bytes): + return repr(httputil.get_ranges('bytes=%s' % bytes, 8)) + + def slice_file(self): + path = os.path.join(os.getcwd(), os.path.dirname(__file__)) + return static.serve_file(os.path.join(path, "static/index.html")) + + + class Cookies(Test): + + def single(self, name): + cookie = cherrypy.request.cookie[name] + # Python2's SimpleCookie.__setitem__ won't take unicode keys. + cherrypy.response.cookie[str(name)] = cookie.value + + def multiple(self, names): + for name in names: + cookie = cherrypy.request.cookie[name] + # Python2's SimpleCookie.__setitem__ won't take unicode keys. + cherrypy.response.cookie[str(name)] = cookie.value + + + cherrypy.tree.mount(root) + setup_server = staticmethod(setup_server) + + + def testStatus(self): + self.getPage("/status/") + self.assertBody('normal') + self.assertStatus(200) + + self.getPage("/status/blank") + self.assertBody('') + self.assertStatus(200) + + self.getPage("/status/illegal") + self.assertStatus(500) + msg = "Illegal response status from server (781 is out of range)." + self.assertErrorPage(500, msg) + + if not getattr(cherrypy.server, 'using_apache', False): + self.getPage("/status/unknown") + self.assertBody('funky') + self.assertStatus(431) + + self.getPage("/status/bad") + self.assertStatus(500) + msg = "Illegal response status from server ('error' is non-numeric)." + self.assertErrorPage(500, msg) + + def testSlashes(self): + # Test that requests for index methods without a trailing slash + # get redirected to the same URI path with a trailing slash. + # Make sure GET params are preserved. + self.getPage("/redirect?id=3") + self.assertStatus(301) + self.assertInBody("<a href='%s/redirect/?id=3'>" + "%s/redirect/?id=3</a>" % (self.base(), self.base())) + + if self.prefix(): + # Corner case: the "trailing slash" redirect could be tricky if + # we're using a virtual root and the URI is "/vroot" (no slash). + self.getPage("") + self.assertStatus(301) + self.assertInBody("<a href='%s/'>%s/</a>" % + (self.base(), self.base())) + + # Test that requests for NON-index methods WITH a trailing slash + # get redirected to the same URI path WITHOUT a trailing slash. + # Make sure GET params are preserved. + self.getPage("/redirect/by_code/?code=307") + self.assertStatus(301) + self.assertInBody("<a href='%s/redirect/by_code?code=307'>" + "%s/redirect/by_code?code=307</a>" + % (self.base(), self.base())) + + # If the trailing_slash tool is off, CP should just continue + # as if the slashes were correct. But it needs some help + # inside cherrypy.url to form correct output. + self.getPage('/url?path_info=page1') + self.assertBody('%s/url/page1' % self.base()) + self.getPage('/url/leaf/?path_info=page1') + self.assertBody('%s/url/page1' % self.base()) + + def testRedirect(self): + self.getPage("/redirect/") + self.assertBody('child') + self.assertStatus(200) + + self.getPage("/redirect/by_code?code=300") + self.assertMatchesBody(r"<a href='(.*)somewhere%20else'>\1somewhere%20else</a>") + self.assertStatus(300) + + self.getPage("/redirect/by_code?code=301") + self.assertMatchesBody(r"<a href='(.*)somewhere%20else'>\1somewhere%20else</a>") + self.assertStatus(301) + + self.getPage("/redirect/by_code?code=302") + self.assertMatchesBody(r"<a href='(.*)somewhere%20else'>\1somewhere%20else</a>") + self.assertStatus(302) + + self.getPage("/redirect/by_code?code=303") + self.assertMatchesBody(r"<a href='(.*)somewhere%20else'>\1somewhere%20else</a>") + self.assertStatus(303) + + self.getPage("/redirect/by_code?code=307") + self.assertMatchesBody(r"<a href='(.*)somewhere%20else'>\1somewhere%20else</a>") + self.assertStatus(307) + + self.getPage("/redirect/nomodify") + self.assertBody('') + self.assertStatus(304) + + self.getPage("/redirect/proxy") + self.assertBody('') + self.assertStatus(305) + + # HTTPRedirect on error + self.getPage("/redirect/error/") + self.assertStatus(('302 Found', '303 See Other')) + self.assertInBody('/errpage') + + # Make sure str(HTTPRedirect()) works. + self.getPage("/redirect/stringify", protocol="HTTP/1.0") + self.assertStatus(200) + self.assertBody("(['%s/'], 302)" % self.base()) + if cherrypy.server.protocol_version == "HTTP/1.1": + self.getPage("/redirect/stringify", protocol="HTTP/1.1") + self.assertStatus(200) + self.assertBody("(['%s/'], 303)" % self.base()) + + # check that #fragments are handled properly + # http://skrb.org/ietf/http_errata.html#location-fragments + frag = "foo" + self.getPage("/redirect/fragment/%s" % frag) + self.assertMatchesBody(r"<a href='(.*)\/some\/url\#%s'>\1\/some\/url\#%s</a>" % (frag, frag)) + loc = self.assertHeader('Location') + assert loc.endswith("#%s" % frag) + self.assertStatus(('302 Found', '303 See Other')) + + # check injection protection + # See http://www.cherrypy.org/ticket/1003 + self.getPage("/redirect/custom?code=303&url=/foobar/%0d%0aSet-Cookie:%20somecookie=someval") + self.assertStatus(303) + loc = self.assertHeader('Location') + assert 'Set-Cookie' in loc + self.assertNoHeader('Set-Cookie') + + def test_InternalRedirect(self): + # InternalRedirect + self.getPage("/internalredirect/") + self.assertBody('hello') + self.assertStatus(200) + + # Test passthrough + self.getPage("/internalredirect/petshop?user_id=Sir-not-appearing-in-this-film") + self.assertBody('0 images for Sir-not-appearing-in-this-film') + self.assertStatus(200) + + # Test args + self.getPage("/internalredirect/petshop?user_id=parrot") + self.assertBody('0 images for slug') + self.assertStatus(200) + + # Test POST + self.getPage("/internalredirect/petshop", method="POST", + body="user_id=terrier") + self.assertBody('0 images for fish') + self.assertStatus(200) + + # Test ir before body read + self.getPage("/internalredirect/early_ir", method="POST", + body="arg=aha!") + self.assertBody("Something went horribly wrong.") + self.assertStatus(200) + + self.getPage("/internalredirect/secure") + self.assertBody('Please log in') + self.assertStatus(200) + + # Relative path in InternalRedirect. + # Also tests request.prev. + self.getPage("/internalredirect/relative?a=3&b=5") + self.assertBody("a=3&b=5") + self.assertStatus(200) + + # InternalRedirect on error + self.getPage("/internalredirect/choke") + self.assertStatus(200) + self.assertBody("Something went horribly wrong.") + + def testFlatten(self): + for url in ["/flatten/as_string", "/flatten/as_list", + "/flatten/as_yield", "/flatten/as_dblyield", + "/flatten/as_refyield"]: + self.getPage(url) + self.assertBody('content') + + def testRanges(self): + self.getPage("/ranges/get_ranges?bytes=3-6") + self.assertBody("[(3, 7)]") + + # Test multiple ranges and a suffix-byte-range-spec, for good measure. + self.getPage("/ranges/get_ranges?bytes=2-4,-1") + self.assertBody("[(2, 5), (7, 8)]") + + # Get a partial file. + if cherrypy.server.protocol_version == "HTTP/1.1": + self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')]) + self.assertStatus(206) + self.assertHeader("Content-Type", "text/html;charset=utf-8") + self.assertHeader("Content-Range", "bytes 2-5/14") + self.assertBody("llo,") + + # What happens with overlapping ranges (and out of order, too)? + self.getPage("/ranges/slice_file", [('Range', 'bytes=4-6,2-5')]) + self.assertStatus(206) + ct = self.assertHeader("Content-Type") + expected_type = "multipart/byteranges; boundary=" + self.assert_(ct.startswith(expected_type)) + boundary = ct[len(expected_type):] + expected_body = ("\r\n--%s\r\n" + "Content-type: text/html\r\n" + "Content-range: bytes 4-6/14\r\n" + "\r\n" + "o, \r\n" + "--%s\r\n" + "Content-type: text/html\r\n" + "Content-range: bytes 2-5/14\r\n" + "\r\n" + "llo,\r\n" + "--%s--\r\n" % (boundary, boundary, boundary)) + self.assertBody(expected_body) + self.assertHeader("Content-Length") + + # Test "416 Requested Range Not Satisfiable" + self.getPage("/ranges/slice_file", [('Range', 'bytes=2300-2900')]) + self.assertStatus(416) + # "When this status code is returned for a byte-range request, + # the response SHOULD include a Content-Range entity-header + # field specifying the current length of the selected resource" + self.assertHeader("Content-Range", "bytes */14") + elif cherrypy.server.protocol_version == "HTTP/1.0": + # Test Range behavior with HTTP/1.0 request + self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')]) + self.assertStatus(200) + self.assertBody("Hello, world\r\n") + + def testFavicon(self): + # favicon.ico is served by staticfile. + icofilename = os.path.join(localDir, "../favicon.ico") + icofile = open(icofilename, "rb") + data = icofile.read() + icofile.close() + + self.getPage("/favicon.ico") + self.assertBody(data) + + def testCookies(self): + if sys.version_info >= (2, 5): + header_value = lambda x: x + else: + header_value = lambda x: x+';' + + self.getPage("/cookies/single?name=First", + [('Cookie', 'First=Dinsdale;')]) + self.assertHeader('Set-Cookie', header_value('First=Dinsdale')) + + self.getPage("/cookies/multiple?names=First&names=Last", + [('Cookie', 'First=Dinsdale; Last=Piranha;'), + ]) + self.assertHeader('Set-Cookie', header_value('First=Dinsdale')) + self.assertHeader('Set-Cookie', header_value('Last=Piranha')) + + self.getPage("/cookies/single?name=Something-With:Colon", + [('Cookie', 'Something-With:Colon=some-value')]) + self.assertStatus(400) + + def testDefaultContentType(self): + self.getPage('/') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.getPage('/defct/plain') + self.getPage('/') + self.assertHeader('Content-Type', 'text/plain;charset=utf-8') + self.getPage('/defct/html') + + def test_cherrypy_url(self): + # Input relative to current + self.getPage('/url/leaf?path_info=page1') + self.assertBody('%s/url/page1' % self.base()) + self.getPage('/url/?path_info=page1') + self.assertBody('%s/url/page1' % self.base()) + # Other host header + host = 'www.mydomain.example' + self.getPage('/url/leaf?path_info=page1', + headers=[('Host', host)]) + self.assertBody('%s://%s/url/page1' % (self.scheme, host)) + + # Input is 'absolute'; that is, relative to script_name + self.getPage('/url/leaf?path_info=/page1') + self.assertBody('%s/page1' % self.base()) + self.getPage('/url/?path_info=/page1') + self.assertBody('%s/page1' % self.base()) + + # Single dots + self.getPage('/url/leaf?path_info=./page1') + self.assertBody('%s/url/page1' % self.base()) + self.getPage('/url/leaf?path_info=other/./page1') + self.assertBody('%s/url/other/page1' % self.base()) + self.getPage('/url/?path_info=/other/./page1') + self.assertBody('%s/other/page1' % self.base()) + + # Double dots + self.getPage('/url/leaf?path_info=../page1') + self.assertBody('%s/page1' % self.base()) + self.getPage('/url/leaf?path_info=other/../page1') + self.assertBody('%s/url/page1' % self.base()) + self.getPage('/url/leaf?path_info=/other/../page1') + self.assertBody('%s/page1' % self.base()) + + # Output relative to current path or script_name + self.getPage('/url/?path_info=page1&relative=True') + self.assertBody('page1') + self.getPage('/url/leaf?path_info=/page1&relative=True') + self.assertBody('../page1') + self.getPage('/url/leaf?path_info=page1&relative=True') + self.assertBody('page1') + self.getPage('/url/leaf?path_info=leaf/page1&relative=True') + self.assertBody('leaf/page1') + self.getPage('/url/leaf?path_info=../page1&relative=True') + self.assertBody('../page1') + self.getPage('/url/?path_info=other/../page1&relative=True') + self.assertBody('page1') + + # Output relative to / + self.getPage('/baseurl?path_info=ab&relative=True') + self.assertBody('ab') + # Output relative to / + self.getPage('/baseurl?path_info=/ab&relative=True') + self.assertBody('ab') + + # absolute-path references ("server-relative") + # Input relative to current + self.getPage('/url/leaf?path_info=page1&relative=server') + self.assertBody('/url/page1') + self.getPage('/url/?path_info=page1&relative=server') + self.assertBody('/url/page1') + # Input is 'absolute'; that is, relative to script_name + self.getPage('/url/leaf?path_info=/page1&relative=server') + self.assertBody('/page1') + self.getPage('/url/?path_info=/page1&relative=server') + self.assertBody('/page1') + + def test_expose_decorator(self): + if not sys.version_info >= (2, 5): + return self.skip("skipped (Python 2.5+ only) ") + + # Test @expose + self.getPage("/expose_dec/no_call") + self.assertStatus(200) + self.assertBody("Mr E. R. Bradshaw") + + # Test @expose() + self.getPage("/expose_dec/call_empty") + self.assertStatus(200) + self.assertBody("Mrs. B.J. Smegma") + + # Test @expose("alias") + self.getPage("/expose_dec/call_alias") + self.assertStatus(200) + self.assertBody("Mr Nesbitt") + # Does the original name work? + self.getPage("/expose_dec/nesbitt") + self.assertStatus(200) + self.assertBody("Mr Nesbitt") + + # Test @expose(["alias1", "alias2"]) + self.getPage("/expose_dec/alias1") + self.assertStatus(200) + self.assertBody("Mr Ken Andrews") + self.getPage("/expose_dec/alias2") + self.assertStatus(200) + self.assertBody("Mr Ken Andrews") + # Does the original name work? + self.getPage("/expose_dec/andrews") + self.assertStatus(200) + self.assertBody("Mr Ken Andrews") + + # Test @expose(alias="alias") + self.getPage("/expose_dec/alias3") + self.assertStatus(200) + self.assertBody("Mr. and Mrs. Watson") + diff --git a/cherrypy/test/test_dynamicobjectmapping.py b/cherrypy/test/test_dynamicobjectmapping.py new file mode 100755 index 0000000..1e04d08 --- /dev/null +++ b/cherrypy/test/test_dynamicobjectmapping.py @@ -0,0 +1,403 @@ +import cherrypy +from cherrypy._cptree import Application +from cherrypy.test import helper + +script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"] + + + +def setup_server(): + class SubSubRoot: + def index(self): + return "SubSubRoot index" + index.exposed = True + + def default(self, *args): + return "SubSubRoot default" + default.exposed = True + + def handler(self): + return "SubSubRoot handler" + handler.exposed = True + + def dispatch(self): + return "SubSubRoot dispatch" + dispatch.exposed = True + + subsubnodes = { + '1': SubSubRoot(), + '2': SubSubRoot(), + } + + class SubRoot: + def index(self): + return "SubRoot index" + index.exposed = True + + def default(self, *args): + return "SubRoot %s" % (args,) + default.exposed = True + + def handler(self): + return "SubRoot handler" + handler.exposed = True + + def _cp_dispatch(self, vpath): + return subsubnodes.get(vpath[0], None) + + subnodes = { + '1': SubRoot(), + '2': SubRoot(), + } + class Root: + def index(self): + return "index" + index.exposed = True + + def default(self, *args): + return "default %s" % (args,) + default.exposed = True + + def handler(self): + return "handler" + handler.exposed = True + + def _cp_dispatch(self, vpath): + return subnodes.get(vpath[0]) + + #-------------------------------------------------------------------------- + # DynamicNodeAndMethodDispatcher example. + # This example exposes a fairly naive HTTP api + class User(object): + def __init__(self, id, name): + self.id = id + self.name = name + + def __unicode__(self): + return unicode(self.name) + + user_lookup = { + 1: User(1, 'foo'), + 2: User(2, 'bar'), + } + + def make_user(name, id=None): + if not id: + id = max(*user_lookup.keys()) + 1 + user_lookup[id] = User(id, name) + return id + + class UserContainerNode(object): + exposed = True + + def POST(self, name): + """ + Allow the creation of a new Object + """ + return "POST %d" % make_user(name) + + def GET(self): + keys = user_lookup.keys() + keys.sort() + return unicode(keys) + + def dynamic_dispatch(self, vpath): + try: + id = int(vpath[0]) + except (ValueError, IndexError): + return None + return UserInstanceNode(id) + + class UserInstanceNode(object): + exposed = True + def __init__(self, id): + self.id = id + self.user = user_lookup.get(id, None) + + # For all but PUT methods there MUST be a valid user identified + # by self.id + if not self.user and cherrypy.request.method != 'PUT': + raise cherrypy.HTTPError(404) + + def GET(self, *args, **kwargs): + """ + Return the appropriate representation of the instance. + """ + return unicode(self.user) + + def POST(self, name): + """ + Update the fields of the user instance. + """ + self.user.name = name + return "POST %d" % self.user.id + + def PUT(self, name): + """ + Create a new user with the specified id, or edit it if it already exists + """ + if self.user: + # Edit the current user + self.user.name = name + return "PUT %d" % self.user.id + else: + # Make a new user with said attributes. + return "PUT %d" % make_user(name, self.id) + + def DELETE(self): + """ + Delete the user specified at the id. + """ + id = self.user.id + del user_lookup[self.user.id] + del self.user + return "DELETE %d" % id + + + class ABHandler: + class CustomDispatch: + def index(self, a, b): + return "custom" + index.exposed = True + + def _cp_dispatch(self, vpath): + """Make sure that if we don't pop anything from vpath, + processing still works. + """ + return self.CustomDispatch() + + def index(self, a, b=None): + body = [ 'a:' + str(a) ] + if b is not None: + body.append(',b:' + str(b)) + return ''.join(body) + index.exposed = True + + def delete(self, a, b): + return 'deleting ' + str(a) + ' and ' + str(b) + delete.exposed = True + + class IndexOnly: + def _cp_dispatch(self, vpath): + """Make sure that popping ALL of vpath still shows the index + handler. + """ + while vpath: + vpath.pop() + return self + + def index(self): + return "IndexOnly index" + index.exposed = True + + class DecoratedPopArgs: + """Test _cp_dispatch with @cherrypy.popargs.""" + def index(self): + return "no params" + index.exposed = True + + def hi(self): + return "hi was not interpreted as 'a' param" + hi.exposed = True + DecoratedPopArgs = cherrypy.popargs('a', 'b', handler=ABHandler())(DecoratedPopArgs) + + class NonDecoratedPopArgs: + """Test _cp_dispatch = cherrypy.popargs()""" + + _cp_dispatch = cherrypy.popargs('a') + + def index(self, a): + return "index: " + str(a) + index.exposed = True + + class ParameterizedHandler: + """Special handler created for each request""" + + def __init__(self, a): + self.a = a + + def index(self): + if 'a' in cherrypy.request.params: + raise Exception("Parameterized handler argument ended up in request.params") + return self.a + index.exposed = True + + class ParameterizedPopArgs: + """Test cherrypy.popargs() with a function call handler""" + ParameterizedPopArgs = cherrypy.popargs('a', handler=ParameterizedHandler)(ParameterizedPopArgs) + + Root.decorated = DecoratedPopArgs() + Root.undecorated = NonDecoratedPopArgs() + Root.index_only = IndexOnly() + Root.parameter_test = ParameterizedPopArgs() + + Root.users = UserContainerNode() + + md = cherrypy.dispatch.MethodDispatcher('dynamic_dispatch') + for url in script_names: + conf = {'/': { + 'user': (url or "/").split("/")[-2], + }, + '/users': { + 'request.dispatch': md + }, + } + cherrypy.tree.mount(Root(), url, conf) + +class DynamicObjectMappingTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def testObjectMapping(self): + for url in script_names: + prefix = self.script_name = url + + self.getPage('/') + self.assertBody('index') + + self.getPage('/handler') + self.assertBody('handler') + + # Dynamic dispatch will succeed here for the subnodes + # so the subroot gets called + self.getPage('/1/') + self.assertBody('SubRoot index') + + self.getPage('/2/') + self.assertBody('SubRoot index') + + self.getPage('/1/handler') + self.assertBody('SubRoot handler') + + self.getPage('/2/handler') + self.assertBody('SubRoot handler') + + # Dynamic dispatch will fail here for the subnodes + # so the default gets called + self.getPage('/asdf/') + self.assertBody("default ('asdf',)") + + self.getPage('/asdf/asdf') + self.assertBody("default ('asdf', 'asdf')") + + self.getPage('/asdf/handler') + self.assertBody("default ('asdf', 'handler')") + + # Dynamic dispatch will succeed here for the subsubnodes + # so the subsubroot gets called + self.getPage('/1/1/') + self.assertBody('SubSubRoot index') + + self.getPage('/2/2/') + self.assertBody('SubSubRoot index') + + self.getPage('/1/1/handler') + self.assertBody('SubSubRoot handler') + + self.getPage('/2/2/handler') + self.assertBody('SubSubRoot handler') + + self.getPage('/2/2/dispatch') + self.assertBody('SubSubRoot dispatch') + + # The exposed dispatch will not be called as a dispatch + # method. + self.getPage('/2/2/foo/foo') + self.assertBody("SubSubRoot default") + + # Dynamic dispatch will fail here for the subsubnodes + # so the SubRoot gets called + self.getPage('/1/asdf/') + self.assertBody("SubRoot ('asdf',)") + + self.getPage('/1/asdf/asdf') + self.assertBody("SubRoot ('asdf', 'asdf')") + + self.getPage('/1/asdf/handler') + self.assertBody("SubRoot ('asdf', 'handler')") + + def testMethodDispatch(self): + # GET acts like a container + self.getPage("/users") + self.assertBody("[1, 2]") + self.assertHeader('Allow', 'GET, HEAD, POST') + + # POST to the container URI allows creation + self.getPage("/users", method="POST", body="name=baz") + self.assertBody("POST 3") + self.assertHeader('Allow', 'GET, HEAD, POST') + + # POST to a specific instanct URI results in a 404 + # as the resource does not exit. + self.getPage("/users/5", method="POST", body="name=baz") + self.assertStatus(404) + + # PUT to a specific instanct URI results in creation + self.getPage("/users/5", method="PUT", body="name=boris") + self.assertBody("PUT 5") + self.assertHeader('Allow', 'DELETE, GET, HEAD, POST, PUT') + + # GET acts like a container + self.getPage("/users") + self.assertBody("[1, 2, 3, 5]") + self.assertHeader('Allow', 'GET, HEAD, POST') + + test_cases = ( + (1, 'foo', 'fooupdated', 'DELETE, GET, HEAD, POST, PUT'), + (2, 'bar', 'barupdated', 'DELETE, GET, HEAD, POST, PUT'), + (3, 'baz', 'bazupdated', 'DELETE, GET, HEAD, POST, PUT'), + (5, 'boris', 'borisupdated', 'DELETE, GET, HEAD, POST, PUT'), + ) + for id, name, updatedname, headers in test_cases: + self.getPage("/users/%d" % id) + self.assertBody(name) + self.assertHeader('Allow', headers) + + # Make sure POSTs update already existings resources + self.getPage("/users/%d" % id, method='POST', body="name=%s" % updatedname) + self.assertBody("POST %d" % id) + self.assertHeader('Allow', headers) + + # Make sure PUTs Update already existing resources. + self.getPage("/users/%d" % id, method='PUT', body="name=%s" % updatedname) + self.assertBody("PUT %d" % id) + self.assertHeader('Allow', headers) + + # Make sure DELETES Remove already existing resources. + self.getPage("/users/%d" % id, method='DELETE') + self.assertBody("DELETE %d" % id) + self.assertHeader('Allow', headers) + + + # GET acts like a container + self.getPage("/users") + self.assertBody("[]") + self.assertHeader('Allow', 'GET, HEAD, POST') + + def testVpathDispatch(self): + self.getPage("/decorated/") + self.assertBody("no params") + + self.getPage("/decorated/hi") + self.assertBody("hi was not interpreted as 'a' param") + + self.getPage("/decorated/yo/") + self.assertBody("a:yo") + + self.getPage("/decorated/yo/there/") + self.assertBody("a:yo,b:there") + + self.getPage("/decorated/yo/there/delete") + self.assertBody("deleting yo and there") + + self.getPage("/decorated/yo/there/handled_by_dispatch/") + self.assertBody("custom") + + self.getPage("/undecorated/blah/") + self.assertBody("index: blah") + + self.getPage("/index_only/a/b/c/d/e/f/g/") + self.assertBody("IndexOnly index") + + self.getPage("/parameter_test/argument2/") + self.assertBody("argument2") + diff --git a/cherrypy/test/test_encoding.py b/cherrypy/test/test_encoding.py new file mode 100755 index 0000000..67b28ed --- /dev/null +++ b/cherrypy/test/test_encoding.py @@ -0,0 +1,363 @@ + +import gzip +import sys + +import cherrypy +from cherrypy._cpcompat import BytesIO, IncompleteRead, ntob, ntou + +europoundUnicode = ntou('\x80\xa3') +sing = u"\u6bdb\u6cfd\u4e1c: Sing, Little Birdie?" +sing8 = sing.encode('utf-8') +sing16 = sing.encode('utf-16') + + +from cherrypy.test import helper + + +class EncodingTests(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self, param): + assert param == europoundUnicode, "%r != %r" % (param, europoundUnicode) + yield europoundUnicode + index.exposed = True + + def mao_zedong(self): + return sing + mao_zedong.exposed = True + + def utf8(self): + return sing8 + utf8.exposed = True + utf8._cp_config = {'tools.encode.encoding': 'utf-8'} + + def cookies_and_headers(self): + # if the headers have non-ascii characters and a cookie has + # any part which is unicode (even ascii), the response + # should not fail. + cherrypy.response.cookie['candy'] = 'bar' + cherrypy.response.cookie['candy']['domain'] = 'cherrypy.org' + cherrypy.response.headers['Some-Header'] = 'My d\xc3\xb6g has fleas' + return 'Any content' + cookies_and_headers.exposed = True + + def reqparams(self, *args, **kwargs): + return ntob(', ').join([": ".join((k, v)).encode('utf8') + for k, v in cherrypy.request.params.items()]) + reqparams.exposed = True + + def nontext(self, *args, **kwargs): + cherrypy.response.headers['Content-Type'] = 'application/binary' + return '\x00\x01\x02\x03' + nontext.exposed = True + nontext._cp_config = {'tools.encode.text_only': False, + 'tools.encode.add_charset': True, + } + + class GZIP: + def index(self): + yield "Hello, world" + index.exposed = True + + def noshow(self): + # Test for ticket #147, where yield showed no exceptions (content- + # encoding was still gzip even though traceback wasn't zipped). + raise IndexError() + yield "Here be dragons" + noshow.exposed = True + # Turn encoding off so the gzip tool is the one doing the collapse. + noshow._cp_config = {'tools.encode.on': False} + + def noshow_stream(self): + # Test for ticket #147, where yield showed no exceptions (content- + # encoding was still gzip even though traceback wasn't zipped). + raise IndexError() + yield "Here be dragons" + noshow_stream.exposed = True + noshow_stream._cp_config = {'response.stream': True} + + class Decode: + def extra_charset(self, *args, **kwargs): + return ', '.join([": ".join((k, v)) + for k, v in cherrypy.request.params.items()]) + extra_charset.exposed = True + extra_charset._cp_config = { + 'tools.decode.on': True, + 'tools.decode.default_encoding': ['utf-16'], + } + + def force_charset(self, *args, **kwargs): + return ', '.join([": ".join((k, v)) + for k, v in cherrypy.request.params.items()]) + force_charset.exposed = True + force_charset._cp_config = { + 'tools.decode.on': True, + 'tools.decode.encoding': 'utf-16', + } + + root = Root() + root.gzip = GZIP() + root.decode = Decode() + cherrypy.tree.mount(root, config={'/gzip': {'tools.gzip.on': True}}) + setup_server = staticmethod(setup_server) + + def test_query_string_decoding(self): + europoundUtf8 = europoundUnicode.encode('utf-8') + self.getPage(ntob('/?param=') + europoundUtf8) + self.assertBody(europoundUtf8) + + # Encoded utf8 query strings MUST be parsed correctly. + # Here, q is the POUND SIGN U+00A3 encoded in utf8 and then %HEX + self.getPage("/reqparams?q=%C2%A3") + # The return value will be encoded as utf8. + self.assertBody(ntob("q: \xc2\xa3")) + + # Query strings that are incorrectly encoded MUST raise 404. + # Here, q is the POUND SIGN U+00A3 encoded in latin1 and then %HEX + self.getPage("/reqparams?q=%A3") + self.assertStatus(404) + self.assertErrorPage(404, + "The given query string could not be processed. Query " + "strings for this resource must be encoded with 'utf8'.") + + def test_urlencoded_decoding(self): + # Test the decoding of an application/x-www-form-urlencoded entity. + europoundUtf8 = europoundUnicode.encode('utf-8') + body=ntob("param=") + europoundUtf8 + self.getPage('/', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(europoundUtf8) + + # Encoded utf8 entities MUST be parsed and decoded correctly. + # Here, q is the POUND SIGN U+00A3 encoded in utf8 + body = ntob("q=\xc2\xa3") + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("q: \xc2\xa3")) + + # ...and in utf16, which is not in the default attempt_charsets list: + body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded;charset=utf-16"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("q: \xc2\xa3")) + + # Entities that are incorrectly encoded MUST raise 400. + # Here, q is the POUND SIGN U+00A3 encoded in utf16, but + # the Content-Type incorrectly labels it utf-8. + body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded;charset=utf-8"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertStatus(400) + self.assertErrorPage(400, + "The request entity could not be decoded. The following charsets " + "were attempted: ['utf-8']") + + def test_decode_tool(self): + # An extra charset should be tried first, and succeed if it matches. + # Here, we add utf-16 as a charset and pass a utf-16 body. + body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") + self.getPage('/decode/extra_charset', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("q: \xc2\xa3")) + + # An extra charset should be tried first, and continue to other default + # charsets if it doesn't match. + # Here, we add utf-16 as a charset but still pass a utf-8 body. + body = ntob("q=\xc2\xa3") + self.getPage('/decode/extra_charset', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("q: \xc2\xa3")) + + # An extra charset should error if force is True and it doesn't match. + # Here, we force utf-16 as a charset but still pass a utf-8 body. + body = ntob("q=\xc2\xa3") + self.getPage('/decode/force_charset', method='POST', + headers=[("Content-Type", "application/x-www-form-urlencoded"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertErrorPage(400, + "The request entity could not be decoded. The following charsets " + "were attempted: ['utf-16']") + + def test_multipart_decoding(self): + # Test the decoding of a multipart entity when the charset (utf16) is + # explicitly given. + body=ntob('\r\n'.join(['--X', + 'Content-Type: text/plain;charset=utf-16', + 'Content-Disposition: form-data; name="text"', + '', + '\xff\xfea\x00b\x00\x1c c\x00', + '--X', + 'Content-Type: text/plain;charset=utf-16', + 'Content-Disposition: form-data; name="submit"', + '', + '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00', + '--X--'])) + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "multipart/form-data;boundary=X"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("text: ab\xe2\x80\x9cc, submit: Create")) + + def test_multipart_decoding_no_charset(self): + # Test the decoding of a multipart entity when the charset (utf8) is + # NOT explicitly given, but is in the list of charsets to attempt. + body=ntob('\r\n'.join(['--X', + 'Content-Disposition: form-data; name="text"', + '', + '\xe2\x80\x9c', + '--X', + 'Content-Disposition: form-data; name="submit"', + '', + 'Create', + '--X--'])) + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "multipart/form-data;boundary=X"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(ntob("text: \xe2\x80\x9c, submit: Create")) + + def test_multipart_decoding_no_successful_charset(self): + # Test the decoding of a multipart entity when the charset (utf16) is + # NOT explicitly given, and is NOT in the list of charsets to attempt. + body=ntob('\r\n'.join(['--X', + 'Content-Disposition: form-data; name="text"', + '', + '\xff\xfea\x00b\x00\x1c c\x00', + '--X', + 'Content-Disposition: form-data; name="submit"', + '', + '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00', + '--X--'])) + self.getPage('/reqparams', method='POST', + headers=[("Content-Type", "multipart/form-data;boundary=X"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertStatus(400) + self.assertErrorPage(400, + "The request entity could not be decoded. The following charsets " + "were attempted: ['us-ascii', 'utf-8']") + + def test_nontext(self): + self.getPage('/nontext') + self.assertHeader('Content-Type', 'application/binary;charset=utf-8') + self.assertBody('\x00\x01\x02\x03') + + def testEncoding(self): + # Default encoding should be utf-8 + self.getPage('/mao_zedong') + self.assertBody(sing8) + + # Ask for utf-16. + self.getPage('/mao_zedong', [('Accept-Charset', 'utf-16')]) + self.assertHeader('Content-Type', 'text/html;charset=utf-16') + self.assertBody(sing16) + + # Ask for multiple encodings. ISO-8859-1 should fail, and utf-16 + # should be produced. + self.getPage('/mao_zedong', [('Accept-Charset', + 'iso-8859-1;q=1, utf-16;q=0.5')]) + self.assertBody(sing16) + + # The "*" value should default to our default_encoding, utf-8 + self.getPage('/mao_zedong', [('Accept-Charset', '*;q=1, utf-7;q=.2')]) + self.assertBody(sing8) + + # Only allow iso-8859-1, which should fail and raise 406. + self.getPage('/mao_zedong', [('Accept-Charset', 'iso-8859-1, *;q=0')]) + self.assertStatus("406 Not Acceptable") + self.assertInBody("Your client sent this Accept-Charset header: " + "iso-8859-1, *;q=0. We tried these charsets: " + "iso-8859-1.") + + # Ask for x-mac-ce, which should be unknown. See ticket #569. + self.getPage('/mao_zedong', [('Accept-Charset', + 'us-ascii, ISO-8859-1, x-mac-ce')]) + self.assertStatus("406 Not Acceptable") + self.assertInBody("Your client sent this Accept-Charset header: " + "us-ascii, ISO-8859-1, x-mac-ce. We tried these " + "charsets: ISO-8859-1, us-ascii, x-mac-ce.") + + # Test the 'encoding' arg to encode. + self.getPage('/utf8') + self.assertBody(sing8) + self.getPage('/utf8', [('Accept-Charset', 'us-ascii, ISO-8859-1')]) + self.assertStatus("406 Not Acceptable") + + def testGzip(self): + zbuf = BytesIO() + zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9) + zfile.write(ntob("Hello, world")) + zfile.close() + + self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip")]) + self.assertInBody(zbuf.getvalue()[:3]) + self.assertHeader("Vary", "Accept-Encoding") + self.assertHeader("Content-Encoding", "gzip") + + # Test when gzip is denied. + self.getPage('/gzip/', headers=[("Accept-Encoding", "identity")]) + self.assertHeader("Vary", "Accept-Encoding") + self.assertNoHeader("Content-Encoding") + self.assertBody("Hello, world") + + self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip;q=0")]) + self.assertHeader("Vary", "Accept-Encoding") + self.assertNoHeader("Content-Encoding") + self.assertBody("Hello, world") + + self.getPage('/gzip/', headers=[("Accept-Encoding", "*;q=0")]) + self.assertStatus(406) + self.assertNoHeader("Content-Encoding") + self.assertErrorPage(406, "identity, gzip") + + # Test for ticket #147 + self.getPage('/gzip/noshow', headers=[("Accept-Encoding", "gzip")]) + self.assertNoHeader('Content-Encoding') + self.assertStatus(500) + self.assertErrorPage(500, pattern="IndexError\n") + + # In this case, there's nothing we can do to deliver a + # readable page, since 1) the gzip header is already set, + # and 2) we may have already written some of the body. + # The fix is to never stream yields when using gzip. + if (cherrypy.server.protocol_version == "HTTP/1.0" or + getattr(cherrypy.server, "using_apache", False)): + self.getPage('/gzip/noshow_stream', + headers=[("Accept-Encoding", "gzip")]) + self.assertHeader('Content-Encoding', 'gzip') + self.assertInBody('\x1f\x8b\x08\x00') + else: + # The wsgiserver will simply stop sending data, and the HTTP client + # will error due to an incomplete chunk-encoded stream. + self.assertRaises((ValueError, IncompleteRead), self.getPage, + '/gzip/noshow_stream', + headers=[("Accept-Encoding", "gzip")]) + + def test_UnicodeHeaders(self): + self.getPage('/cookies_and_headers') + self.assertBody('Any content') + diff --git a/cherrypy/test/test_etags.py b/cherrypy/test/test_etags.py new file mode 100755 index 0000000..026f9d6 --- /dev/null +++ b/cherrypy/test/test_etags.py @@ -0,0 +1,81 @@ +import cherrypy +from cherrypy.test import helper + + +class ETagTest(helper.CPWebCase): + + def setup_server(): + class Root: + def resource(self): + return "Oh wah ta goo Siam." + resource.exposed = True + + def fail(self, code): + code = int(code) + if 300 <= code <= 399: + raise cherrypy.HTTPRedirect([], code) + else: + raise cherrypy.HTTPError(code) + fail.exposed = True + + def unicoded(self): + return u'I am a \u1ee4nicode string.' + unicoded.exposed = True + unicoded._cp_config = {'tools.encode.on': True} + + conf = {'/': {'tools.etags.on': True, + 'tools.etags.autotags': True, + }} + cherrypy.tree.mount(Root(), config=conf) + setup_server = staticmethod(setup_server) + + def test_etags(self): + self.getPage("/resource") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.assertBody('Oh wah ta goo Siam.') + etag = self.assertHeader('ETag') + + # Test If-Match (both valid and invalid) + self.getPage("/resource", headers=[('If-Match', etag)]) + self.assertStatus("200 OK") + self.getPage("/resource", headers=[('If-Match', "*")]) + self.assertStatus("200 OK") + self.getPage("/resource", headers=[('If-Match', "*")], method="POST") + self.assertStatus("200 OK") + self.getPage("/resource", headers=[('If-Match', "a bogus tag")]) + self.assertStatus("412 Precondition Failed") + + # Test If-None-Match (both valid and invalid) + self.getPage("/resource", headers=[('If-None-Match', etag)]) + self.assertStatus(304) + self.getPage("/resource", method='POST', headers=[('If-None-Match', etag)]) + self.assertStatus("412 Precondition Failed") + self.getPage("/resource", headers=[('If-None-Match', "*")]) + self.assertStatus(304) + self.getPage("/resource", headers=[('If-None-Match', "a bogus tag")]) + self.assertStatus("200 OK") + + def test_errors(self): + self.getPage("/resource") + self.assertStatus(200) + etag = self.assertHeader('ETag') + + # Test raising errors in page handler + self.getPage("/fail/412", headers=[('If-Match', etag)]) + self.assertStatus(412) + self.getPage("/fail/304", headers=[('If-Match', etag)]) + self.assertStatus(304) + self.getPage("/fail/412", headers=[('If-None-Match', "*")]) + self.assertStatus(412) + self.getPage("/fail/304", headers=[('If-None-Match', "*")]) + self.assertStatus(304) + + def test_unicode_body(self): + self.getPage("/unicoded") + self.assertStatus(200) + etag1 = self.assertHeader('ETag') + self.getPage("/unicoded", headers=[('If-Match', etag1)]) + self.assertStatus(200) + self.assertHeader('ETag', etag1) + diff --git a/cherrypy/test/test_http.py b/cherrypy/test/test_http.py new file mode 100755 index 0000000..eb72b5b --- /dev/null +++ b/cherrypy/test/test_http.py @@ -0,0 +1,168 @@ +"""Tests for managing HTTP issues (malformed requests, etc).""" + +import mimetypes + +import cherrypy +from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob + + +def encode_multipart_formdata(files): + """Return (content_type, body) ready for httplib.HTTP instance. + + files: a sequence of (name, filename, value) tuples for multipart uploads. + """ + BOUNDARY = '________ThIs_Is_tHe_bouNdaRY_$' + L = [] + for key, filename, value in files: + L.append('--' + BOUNDARY) + L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % + (key, filename)) + ct = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + L.append('Content-Type: %s' % ct) + L.append('') + L.append(value) + L.append('--' + BOUNDARY + '--') + L.append('') + body = '\r\n'.join(L) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body + + + + +from cherrypy.test import helper + +class HTTPTests(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self, *args, **kwargs): + return "Hello world!" + index.exposed = True + + def no_body(self, *args, **kwargs): + return "Hello world!" + no_body.exposed = True + no_body._cp_config = {'request.process_request_body': False} + + def post_multipart(self, file): + """Return a summary ("a * 65536\nb * 65536") of the uploaded file.""" + contents = file.file.read() + summary = [] + curchar = "" + count = 0 + for c in contents: + if c == curchar: + count += 1 + else: + if count: + summary.append("%s * %d" % (curchar, count)) + count = 1 + curchar = c + if count: + summary.append("%s * %d" % (curchar, count)) + return ", ".join(summary) + post_multipart.exposed = True + + cherrypy.tree.mount(Root()) + cherrypy.config.update({'server.max_request_body_size': 30000000}) + setup_server = staticmethod(setup_server) + + def test_no_content_length(self): + # "The presence of a message-body in a request is signaled by the + # inclusion of a Content-Length or Transfer-Encoding header field in + # the request's message-headers." + # + # Send a message with neither header and no body. Even though + # the request is of method POST, this should be OK because we set + # request.process_request_body to False for our handler. + if self.scheme == "https": + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + c.request("POST", "/no_body") + response = c.getresponse() + self.body = response.fp.read() + self.status = str(response.status) + self.assertStatus(200) + self.assertBody(ntob('Hello world!')) + + # Now send a message that has no Content-Length, but does send a body. + # Verify that CP times out the socket and responds + # with 411 Length Required. + if self.scheme == "https": + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + c.request("POST", "/") + response = c.getresponse() + self.body = response.fp.read() + self.status = str(response.status) + self.assertStatus(411) + + def test_post_multipart(self): + alphabet = "abcdefghijklmnopqrstuvwxyz" + # generate file contents for a large post + contents = "".join([c * 65536 for c in alphabet]) + + # encode as multipart form data + files=[('file', 'file.txt', contents)] + content_type, body = encode_multipart_formdata(files) + body = body.encode('Latin-1') + + # post file + if self.scheme == 'https': + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + c.putrequest('POST', '/post_multipart') + c.putheader('Content-Type', content_type) + c.putheader('Content-Length', str(len(body))) + c.endheaders() + c.send(body) + + response = c.getresponse() + self.body = response.fp.read() + self.status = str(response.status) + self.assertStatus(200) + self.assertBody(", ".join(["%s * 65536" % c for c in alphabet])) + + def test_malformed_request_line(self): + if getattr(cherrypy.server, "using_apache", False): + return self.skip("skipped due to known Apache differences...") + + # Test missing version in Request-Line + if self.scheme == 'https': + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + c._output(ntob('GET /')) + c._send_output() + if hasattr(c, 'strict'): + response = c.response_class(c.sock, strict=c.strict, method='GET') + else: + # Python 3.2 removed the 'strict' feature, saying: + # "http.client now always assumes HTTP/1.x compliant servers." + response = c.response_class(c.sock, method='GET') + response.begin() + self.assertEqual(response.status, 400) + self.assertEqual(response.fp.read(22), ntob("Malformed Request-Line")) + c.close() + + def test_malformed_header(self): + if self.scheme == 'https': + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + c.putrequest('GET', '/') + c.putheader('Content-Type', 'text/plain') + # See http://www.cherrypy.org/ticket/941 + c._output(ntob('Re, 1.2.3.4#015#012')) + c.endheaders() + + response = c.getresponse() + self.status = str(response.status) + self.assertStatus(400) + self.body = response.fp.read(20) + self.assertBody("Illegal header line.") + diff --git a/cherrypy/test/test_httpauth.py b/cherrypy/test/test_httpauth.py new file mode 100755 index 0000000..9d0eecb --- /dev/null +++ b/cherrypy/test/test_httpauth.py @@ -0,0 +1,151 @@ +import cherrypy +from cherrypy._cpcompat import md5, sha, ntob +from cherrypy.lib import httpauth + +from cherrypy.test import helper + +class HTTPAuthTest(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self): + return "This is public." + index.exposed = True + + class DigestProtected: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + class BasicProtected: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + class BasicProtected2: + def index(self): + return "Hello %s, you've been authorized." % cherrypy.request.login + index.exposed = True + + def fetch_users(): + return {'test': 'test'} + + def sha_password_encrypter(password): + return sha(ntob(password)).hexdigest() + + def fetch_password(username): + return sha(ntob('test')).hexdigest() + + conf = {'/digest': {'tools.digest_auth.on': True, + 'tools.digest_auth.realm': 'localhost', + 'tools.digest_auth.users': fetch_users}, + '/basic': {'tools.basic_auth.on': True, + 'tools.basic_auth.realm': 'localhost', + 'tools.basic_auth.users': {'test': md5(ntob('test')).hexdigest()}}, + '/basic2': {'tools.basic_auth.on': True, + 'tools.basic_auth.realm': 'localhost', + 'tools.basic_auth.users': fetch_password, + 'tools.basic_auth.encrypt': sha_password_encrypter}} + + root = Root() + root.digest = DigestProtected() + root.basic = BasicProtected() + root.basic2 = BasicProtected2() + cherrypy.tree.mount(root, config=conf) + setup_server = staticmethod(setup_server) + + + def testPublic(self): + self.getPage("/") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.assertBody('This is public.') + + def testBasic(self): + self.getPage("/basic/") + self.assertStatus(401) + self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"') + + self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZX60')]) + self.assertStatus(401) + + self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZXN0')]) + self.assertStatus('200 OK') + self.assertBody("Hello test, you've been authorized.") + + def testBasic2(self): + self.getPage("/basic2/") + self.assertStatus(401) + self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"') + + self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZX60')]) + self.assertStatus(401) + + self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZXN0')]) + self.assertStatus('200 OK') + self.assertBody("Hello test, you've been authorized.") + + def testDigest(self): + self.getPage("/digest/") + self.assertStatus(401) + + value = None + for k, v in self.headers: + if k.lower() == "www-authenticate": + if v.startswith("Digest"): + value = v + break + + if value is None: + self._handlewebError("Digest authentification scheme was not found") + + value = value[7:] + items = value.split(', ') + tokens = {} + for item in items: + key, value = item.split('=') + tokens[key.lower()] = value + + missing_msg = "%s is missing" + bad_value_msg = "'%s' was expecting '%s' but found '%s'" + nonce = None + if 'realm' not in tokens: + self._handlewebError(missing_msg % 'realm') + elif tokens['realm'] != '"localhost"': + self._handlewebError(bad_value_msg % ('realm', '"localhost"', tokens['realm'])) + if 'nonce' not in tokens: + self._handlewebError(missing_msg % 'nonce') + else: + nonce = tokens['nonce'].strip('"') + if 'algorithm' not in tokens: + self._handlewebError(missing_msg % 'algorithm') + elif tokens['algorithm'] != '"MD5"': + self._handlewebError(bad_value_msg % ('algorithm', '"MD5"', tokens['algorithm'])) + if 'qop' not in tokens: + self._handlewebError(missing_msg % 'qop') + elif tokens['qop'] != '"auth"': + self._handlewebError(bad_value_msg % ('qop', '"auth"', tokens['qop'])) + + # Test a wrong 'realm' value + base_auth = 'Digest username="test", realm="wrong realm", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' + + auth = base_auth % (nonce, '', '00000001') + params = httpauth.parseAuthorization(auth) + response = httpauth._computeDigestResponse(params, 'test') + + auth = base_auth % (nonce, response, '00000001') + self.getPage('/digest/', [('Authorization', auth)]) + self.assertStatus(401) + + # Test that must pass + base_auth = 'Digest username="test", realm="localhost", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' + + auth = base_auth % (nonce, '', '00000001') + params = httpauth.parseAuthorization(auth) + response = httpauth._computeDigestResponse(params, 'test') + + auth = base_auth % (nonce, response, '00000001') + self.getPage('/digest/', [('Authorization', auth)]) + self.assertStatus('200 OK') + self.assertBody("Hello test, you've been authorized.") + diff --git a/cherrypy/test/test_httplib.py b/cherrypy/test/test_httplib.py new file mode 100755 index 0000000..5dc40fd --- /dev/null +++ b/cherrypy/test/test_httplib.py @@ -0,0 +1,29 @@ +"""Tests for cherrypy/lib/httputil.py.""" + +import unittest +from cherrypy.lib import httputil + + +class UtilityTests(unittest.TestCase): + + def test_urljoin(self): + # Test all slash+atom combinations for SCRIPT_NAME and PATH_INFO + self.assertEqual(httputil.urljoin("/sn/", "/pi/"), "/sn/pi/") + self.assertEqual(httputil.urljoin("/sn/", "/pi"), "/sn/pi") + self.assertEqual(httputil.urljoin("/sn/", "/"), "/sn/") + self.assertEqual(httputil.urljoin("/sn/", ""), "/sn/") + self.assertEqual(httputil.urljoin("/sn", "/pi/"), "/sn/pi/") + self.assertEqual(httputil.urljoin("/sn", "/pi"), "/sn/pi") + self.assertEqual(httputil.urljoin("/sn", "/"), "/sn/") + self.assertEqual(httputil.urljoin("/sn", ""), "/sn") + self.assertEqual(httputil.urljoin("/", "/pi/"), "/pi/") + self.assertEqual(httputil.urljoin("/", "/pi"), "/pi") + self.assertEqual(httputil.urljoin("/", "/"), "/") + self.assertEqual(httputil.urljoin("/", ""), "/") + self.assertEqual(httputil.urljoin("", "/pi/"), "/pi/") + self.assertEqual(httputil.urljoin("", "/pi"), "/pi") + self.assertEqual(httputil.urljoin("", "/"), "/") + self.assertEqual(httputil.urljoin("", ""), "/") + +if __name__ == '__main__': + unittest.main() diff --git a/cherrypy/test/test_json.py b/cherrypy/test/test_json.py new file mode 100755 index 0000000..a02c076 --- /dev/null +++ b/cherrypy/test/test_json.py @@ -0,0 +1,79 @@ +import cherrypy +from cherrypy.test import helper + +from cherrypy._cpcompat import json + +class JsonTest(helper.CPWebCase): + def setup_server(): + class Root(object): + def plain(self): + return 'hello' + plain.exposed = True + + def json_string(self): + return 'hello' + json_string.exposed = True + json_string._cp_config = {'tools.json_out.on': True} + + def json_list(self): + return ['a', 'b', 42] + json_list.exposed = True + json_list._cp_config = {'tools.json_out.on': True} + + def json_dict(self): + return {'answer': 42} + json_dict.exposed = True + json_dict._cp_config = {'tools.json_out.on': True} + + def json_post(self): + if cherrypy.request.json == [13, 'c']: + return 'ok' + else: + return 'nok' + json_post.exposed = True + json_post._cp_config = {'tools.json_in.on': True} + + root = Root() + cherrypy.tree.mount(root) + setup_server = staticmethod(setup_server) + + def test_json_output(self): + if json is None: + self.skip("json not found ") + return + + self.getPage("/plain") + self.assertBody("hello") + + self.getPage("/json_string") + self.assertBody('"hello"') + + self.getPage("/json_list") + self.assertBody('["a", "b", 42]') + + self.getPage("/json_dict") + self.assertBody('{"answer": 42}') + + def test_json_input(self): + if json is None: + self.skip("json not found ") + return + + body = '[13, "c"]' + headers = [('Content-Type', 'application/json'), + ('Content-Length', str(len(body)))] + self.getPage("/json_post", method="POST", headers=headers, body=body) + self.assertBody('ok') + + body = '[13, "c"]' + headers = [('Content-Type', 'text/plain'), + ('Content-Length', str(len(body)))] + self.getPage("/json_post", method="POST", headers=headers, body=body) + self.assertStatus(415, 'Expected an application/json content type') + + body = '[13, -]' + headers = [('Content-Type', 'application/json'), + ('Content-Length', str(len(body)))] + self.getPage("/json_post", method="POST", headers=headers, body=body) + self.assertStatus(400, 'Invalid JSON document') + diff --git a/cherrypy/test/test_logging.py b/cherrypy/test/test_logging.py new file mode 100755 index 0000000..5a13cd4 --- /dev/null +++ b/cherrypy/test/test_logging.py @@ -0,0 +1,149 @@ +"""Basic tests for the CherryPy core: request handling.""" + +import os +localDir = os.path.dirname(__file__) + +import cherrypy + +access_log = os.path.join(localDir, "access.log") +error_log = os.path.join(localDir, "error.log") + +# Some unicode strings. +tartaros = u'\u03a4\u1f71\u03c1\u03c4\u03b1\u03c1\u03bf\u03c2' +erebos = u'\u0388\u03c1\u03b5\u03b2\u03bf\u03c2.com' + + +def setup_server(): + class Root: + + def index(self): + return "hello" + index.exposed = True + + def uni_code(self): + cherrypy.request.login = tartaros + cherrypy.request.remote.name = erebos + uni_code.exposed = True + + def slashes(self): + cherrypy.request.request_line = r'GET /slashed\path HTTP/1.1' + slashes.exposed = True + + def whitespace(self): + # User-Agent = "User-Agent" ":" 1*( product | comment ) + # comment = "(" *( ctext | quoted-pair | comment ) ")" + # ctext = <any TEXT excluding "(" and ")"> + # TEXT = <any OCTET except CTLs, but including LWS> + # LWS = [CRLF] 1*( SP | HT ) + cherrypy.request.headers['User-Agent'] = 'Browzuh (1.0\r\n\t\t.3)' + whitespace.exposed = True + + def as_string(self): + return "content" + as_string.exposed = True + + def as_yield(self): + yield "content" + as_yield.exposed = True + + def error(self): + raise ValueError() + error.exposed = True + error._cp_config = {'tools.log_tracebacks.on': True} + + root = Root() + + + cherrypy.config.update({'log.error_file': error_log, + 'log.access_file': access_log, + }) + cherrypy.tree.mount(root) + + + +from cherrypy.test import helper, logtest + +class AccessLogTests(helper.CPWebCase, logtest.LogCase): + setup_server = staticmethod(setup_server) + + logfile = access_log + + def testNormalReturn(self): + self.markLog() + self.getPage("/as_string", + headers=[('Referer', 'http://www.cherrypy.org/'), + ('User-Agent', 'Mozilla/5.0')]) + self.assertBody('content') + self.assertStatus(200) + + intro = '%s - - [' % self.interface() + + self.assertLog(-1, intro) + + if [k for k, v in self.headers if k.lower() == 'content-length']: + self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 7 ' + '"http://www.cherrypy.org/" "Mozilla/5.0"' + % self.prefix()) + else: + self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 - ' + '"http://www.cherrypy.org/" "Mozilla/5.0"' + % self.prefix()) + + def testNormalYield(self): + self.markLog() + self.getPage("/as_yield") + self.assertBody('content') + self.assertStatus(200) + + intro = '%s - - [' % self.interface() + + self.assertLog(-1, intro) + if [k for k, v in self.headers if k.lower() == 'content-length']: + self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 7 "" ""' % + self.prefix()) + else: + self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 - "" ""' + % self.prefix()) + + def testEscapedOutput(self): + # Test unicode in access log pieces. + self.markLog() + self.getPage("/uni_code") + self.assertStatus(200) + self.assertLog(-1, repr(tartaros.encode('utf8'))[1:-1]) + # Test the erebos value. Included inline for your enlightenment. + # Note the 'r' prefix--those backslashes are literals. + self.assertLog(-1, r'\xce\x88\xcf\x81\xce\xb5\xce\xb2\xce\xbf\xcf\x82') + + # Test backslashes in output. + self.markLog() + self.getPage("/slashes") + self.assertStatus(200) + self.assertLog(-1, r'"GET /slashed\\path HTTP/1.1"') + + # Test whitespace in output. + self.markLog() + self.getPage("/whitespace") + self.assertStatus(200) + # Again, note the 'r' prefix. + self.assertLog(-1, r'"Browzuh (1.0\r\n\t\t.3)"') + + +class ErrorLogTests(helper.CPWebCase, logtest.LogCase): + setup_server = staticmethod(setup_server) + + logfile = error_log + + def testTracebacks(self): + # Test that tracebacks get written to the error log. + self.markLog() + ignore = helper.webtest.ignored_exceptions + ignore.append(ValueError) + try: + self.getPage("/error") + self.assertInBody("raise ValueError()") + self.assertLog(0, 'HTTP Traceback (most recent call last):') + self.assertLog(-3, 'raise ValueError()') + finally: + ignore.pop() + diff --git a/cherrypy/test/test_mime.py b/cherrypy/test/test_mime.py new file mode 100755 index 0000000..605071b --- /dev/null +++ b/cherrypy/test/test_mime.py @@ -0,0 +1,128 @@ +"""Tests for various MIME issues, including the safe_multipart Tool.""" + +import cherrypy +from cherrypy._cpcompat import ntob, ntou, sorted + +def setup_server(): + + class Root: + + def multipart(self, parts): + return repr(parts) + multipart.exposed = True + + def multipart_form_data(self, **kwargs): + return repr(list(sorted(kwargs.items()))) + multipart_form_data.exposed = True + + def flashupload(self, Filedata, Upload, Filename): + return ("Upload: %r, Filename: %r, Filedata: %r" % + (Upload, Filename, Filedata.file.read())) + flashupload.exposed = True + + cherrypy.config.update({'server.max_request_body_size': 0}) + cherrypy.tree.mount(Root()) + + +# Client-side code # + +from cherrypy.test import helper + +class MultipartTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_multipart(self): + text_part = ntou("This is the text version") + html_part = ntou("""<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> + <meta content="text/html;charset=ISO-8859-1" http-equiv="Content-Type"> +</head> +<body bgcolor="#ffffff" text="#000000"> + +This is the <strong>HTML</strong> version +</body> +</html> +""") + body = '\r\n'.join([ + "--123456789", + "Content-Type: text/plain; charset='ISO-8859-1'", + "Content-Transfer-Encoding: 7bit", + "", + text_part, + "--123456789", + "Content-Type: text/html; charset='ISO-8859-1'", + "", + html_part, + "--123456789--"]) + headers = [ + ('Content-Type', 'multipart/mixed; boundary=123456789'), + ('Content-Length', str(len(body))), + ] + self.getPage('/multipart', headers, "POST", body) + self.assertBody(repr([text_part, html_part])) + + def test_multipart_form_data(self): + body='\r\n'.join(['--X', + 'Content-Disposition: form-data; name="foo"', + '', + 'bar', + '--X', + # Test a param with more than one value. + # See http://www.cherrypy.org/ticket/1028 + 'Content-Disposition: form-data; name="baz"', + '', + '111', + '--X', + 'Content-Disposition: form-data; name="baz"', + '', + '333', + '--X--']) + self.getPage('/multipart_form_data', method='POST', + headers=[("Content-Type", "multipart/form-data;boundary=X"), + ("Content-Length", str(len(body))), + ], + body=body), + self.assertBody(repr([('baz', [u'111', u'333']), ('foo', u'bar')])) + + +class SafeMultipartHandlingTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_Flash_Upload(self): + headers = [ + ('Accept', 'text/*'), + ('Content-Type', 'multipart/form-data; ' + 'boundary=----------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6'), + ('User-Agent', 'Shockwave Flash'), + ('Host', 'www.example.com:8080'), + ('Content-Length', '499'), + ('Connection', 'Keep-Alive'), + ('Cache-Control', 'no-cache'), + ] + filedata = ntob('<?xml version="1.0" encoding="UTF-8"?>\r\n' + '<projectDescription>\r\n' + '</projectDescription>\r\n') + body = (ntob( + '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' + 'Content-Disposition: form-data; name="Filename"\r\n' + '\r\n' + '.project\r\n' + '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' + 'Content-Disposition: form-data; ' + 'name="Filedata"; filename=".project"\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n') + + filedata + + ntob('\r\n' + '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' + 'Content-Disposition: form-data; name="Upload"\r\n' + '\r\n' + 'Submit Query\r\n' + # Flash apps omit the trailing \r\n on the last line: + '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6--' + )) + self.getPage('/flashupload', headers, "POST", body) + self.assertBody("Upload: u'Submit Query', Filename: u'.project', " + "Filedata: %r" % filedata) + diff --git a/cherrypy/test/test_misc_tools.py b/cherrypy/test/test_misc_tools.py new file mode 100755 index 0000000..fb94e86 --- /dev/null +++ b/cherrypy/test/test_misc_tools.py @@ -0,0 +1,202 @@ +import os +localDir = os.path.dirname(__file__) +logfile = os.path.join(localDir, "test_misc_tools.log") + +import cherrypy +from cherrypy import tools + + +def setup_server(): + class Root: + def index(self): + yield "Hello, world" + index.exposed = True + h = [("Content-Language", "en-GB"), ('Content-Type', 'text/plain')] + tools.response_headers(headers=h)(index) + + def other(self): + return "salut" + other.exposed = True + other._cp_config = { + 'tools.response_headers.on': True, + 'tools.response_headers.headers': [("Content-Language", "fr"), + ('Content-Type', 'text/plain')], + 'tools.log_hooks.on': True, + } + + + class Accept: + _cp_config = {'tools.accept.on': True} + + def index(self): + return '<a href="feed">Atom feed</a>' + index.exposed = True + + # In Python 2.4+, we could use a decorator instead: + # @tools.accept('application/atom+xml') + def feed(self): + return """<?xml version="1.0" encoding="utf-8"?> +<feed xmlns="http://www.w3.org/2005/Atom"> + <title>Unknown Blog</title> +</feed>""" + feed.exposed = True + feed._cp_config = {'tools.accept.media': 'application/atom+xml'} + + def select(self): + # We could also write this: mtype = cherrypy.lib.accept.accept(...) + mtype = tools.accept.callable(['text/html', 'text/plain']) + if mtype == 'text/html': + return "<h2>Page Title</h2>" + else: + return "PAGE TITLE" + select.exposed = True + + class Referer: + def accept(self): + return "Accepted!" + accept.exposed = True + reject = accept + + class AutoVary: + def index(self): + # Read a header directly with 'get' + ae = cherrypy.request.headers.get('Accept-Encoding') + # Read a header directly with '__getitem__' + cl = cherrypy.request.headers['Host'] + # Read a header directly with '__contains__' + hasif = 'If-Modified-Since' in cherrypy.request.headers + # Read a header directly with 'has_key' + has = cherrypy.request.headers.has_key('Range') + # Call a lib function + mtype = tools.accept.callable(['text/html', 'text/plain']) + return "Hello, world!" + index.exposed = True + + conf = {'/referer': {'tools.referer.on': True, + 'tools.referer.pattern': r'http://[^/]*example\.com', + }, + '/referer/reject': {'tools.referer.accept': False, + 'tools.referer.accept_missing': True, + }, + '/autovary': {'tools.autovary.on': True}, + } + + root = Root() + root.referer = Referer() + root.accept = Accept() + root.autovary = AutoVary() + cherrypy.tree.mount(root, config=conf) + cherrypy.config.update({'log.error_file': logfile}) + + +from cherrypy.test import helper + +class ResponseHeadersTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def testResponseHeadersDecorator(self): + self.getPage('/') + self.assertHeader("Content-Language", "en-GB") + self.assertHeader('Content-Type', 'text/plain;charset=utf-8') + + def testResponseHeaders(self): + self.getPage('/other') + self.assertHeader("Content-Language", "fr") + self.assertHeader('Content-Type', 'text/plain;charset=utf-8') + + +class RefererTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def testReferer(self): + self.getPage('/referer/accept') + self.assertErrorPage(403, 'Forbidden Referer header.') + + self.getPage('/referer/accept', + headers=[('Referer', 'http://www.example.com/')]) + self.assertStatus(200) + self.assertBody('Accepted!') + + # Reject + self.getPage('/referer/reject') + self.assertStatus(200) + self.assertBody('Accepted!') + + self.getPage('/referer/reject', + headers=[('Referer', 'http://www.example.com/')]) + self.assertErrorPage(403, 'Forbidden Referer header.') + + +class AcceptTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_Accept_Tool(self): + # Test with no header provided + self.getPage('/accept/feed') + self.assertStatus(200) + self.assertInBody('<title>Unknown Blog</title>') + + # Specify exact media type + self.getPage('/accept/feed', headers=[('Accept', 'application/atom+xml')]) + self.assertStatus(200) + self.assertInBody('<title>Unknown Blog</title>') + + # Specify matching media range + self.getPage('/accept/feed', headers=[('Accept', 'application/*')]) + self.assertStatus(200) + self.assertInBody('<title>Unknown Blog</title>') + + # Specify all media ranges + self.getPage('/accept/feed', headers=[('Accept', '*/*')]) + self.assertStatus(200) + self.assertInBody('<title>Unknown Blog</title>') + + # Specify unacceptable media types + self.getPage('/accept/feed', headers=[('Accept', 'text/html')]) + self.assertErrorPage(406, + "Your client sent this Accept header: text/html. " + "But this resource only emits these media types: " + "application/atom+xml.") + + # Test resource where tool is 'on' but media is None (not set). + self.getPage('/accept/') + self.assertStatus(200) + self.assertBody('<a href="feed">Atom feed</a>') + + def test_accept_selection(self): + # Try both our expected media types + self.getPage('/accept/select', [('Accept', 'text/html')]) + self.assertStatus(200) + self.assertBody('<h2>Page Title</h2>') + self.getPage('/accept/select', [('Accept', 'text/plain')]) + self.assertStatus(200) + self.assertBody('PAGE TITLE') + self.getPage('/accept/select', [('Accept', 'text/plain, text/*;q=0.5')]) + self.assertStatus(200) + self.assertBody('PAGE TITLE') + + # text/* and */* should prefer text/html since it comes first + # in our 'media' argument to tools.accept + self.getPage('/accept/select', [('Accept', 'text/*')]) + self.assertStatus(200) + self.assertBody('<h2>Page Title</h2>') + self.getPage('/accept/select', [('Accept', '*/*')]) + self.assertStatus(200) + self.assertBody('<h2>Page Title</h2>') + + # Try unacceptable media types + self.getPage('/accept/select', [('Accept', 'application/xml')]) + self.assertErrorPage(406, + "Your client sent this Accept header: application/xml. " + "But this resource only emits these media types: " + "text/html, text/plain.") + + +class AutoVaryTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def testAutoVary(self): + self.getPage('/autovary/') + self.assertHeader( + "Vary", 'Accept, Accept-Charset, Accept-Encoding, Host, If-Modified-Since, Range') + diff --git a/cherrypy/test/test_objectmapping.py b/cherrypy/test/test_objectmapping.py new file mode 100755 index 0000000..46816fc --- /dev/null +++ b/cherrypy/test/test_objectmapping.py @@ -0,0 +1,403 @@ +import cherrypy +from cherrypy._cptree import Application +from cherrypy.test import helper + +script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"] + + +class ObjectMappingTest(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self, name="world"): + return name + index.exposed = True + + def foobar(self): + return "bar" + foobar.exposed = True + + def default(self, *params, **kwargs): + return "default:" + repr(params) + default.exposed = True + + def other(self): + return "other" + other.exposed = True + + def extra(self, *p): + return repr(p) + extra.exposed = True + + def redirect(self): + raise cherrypy.HTTPRedirect('dir1/', 302) + redirect.exposed = True + + def notExposed(self): + return "not exposed" + + def confvalue(self): + return cherrypy.request.config.get("user") + confvalue.exposed = True + + def redirect_via_url(self, path): + raise cherrypy.HTTPRedirect(cherrypy.url(path)) + redirect_via_url.exposed = True + + def translate_html(self): + return "OK" + translate_html.exposed = True + + def mapped_func(self, ID=None): + return "ID is %s" % ID + mapped_func.exposed = True + setattr(Root, "Von B\xfclow", mapped_func) + + + class Exposing: + def base(self): + return "expose works!" + cherrypy.expose(base) + cherrypy.expose(base, "1") + cherrypy.expose(base, "2") + + class ExposingNewStyle(object): + def base(self): + return "expose works!" + cherrypy.expose(base) + cherrypy.expose(base, "1") + cherrypy.expose(base, "2") + + + class Dir1: + def index(self): + return "index for dir1" + index.exposed = True + + def myMethod(self): + return "myMethod from dir1, path_info is:" + repr(cherrypy.request.path_info) + myMethod.exposed = True + myMethod._cp_config = {'tools.trailing_slash.extra': True} + + def default(self, *params): + return "default for dir1, param is:" + repr(params) + default.exposed = True + + + class Dir2: + def index(self): + return "index for dir2, path is:" + cherrypy.request.path_info + index.exposed = True + + def script_name(self): + return cherrypy.tree.script_name() + script_name.exposed = True + + def cherrypy_url(self): + return cherrypy.url("/extra") + cherrypy_url.exposed = True + + def posparam(self, *vpath): + return "/".join(vpath) + posparam.exposed = True + + + class Dir3: + def default(self): + return "default for dir3, not exposed" + + class Dir4: + def index(self): + return "index for dir4, not exposed" + + class DefNoIndex: + def default(self, *args): + raise cherrypy.HTTPRedirect("contact") + default.exposed = True + + # MethodDispatcher code + class ByMethod: + exposed = True + + def __init__(self, *things): + self.things = list(things) + + def GET(self): + return repr(self.things) + + def POST(self, thing): + self.things.append(thing) + + class Collection: + default = ByMethod('a', 'bit') + + Root.exposing = Exposing() + Root.exposingnew = ExposingNewStyle() + Root.dir1 = Dir1() + Root.dir1.dir2 = Dir2() + Root.dir1.dir2.dir3 = Dir3() + Root.dir1.dir2.dir3.dir4 = Dir4() + Root.defnoindex = DefNoIndex() + Root.bymethod = ByMethod('another') + Root.collection = Collection() + + d = cherrypy.dispatch.MethodDispatcher() + for url in script_names: + conf = {'/': {'user': (url or "/").split("/")[-2]}, + '/bymethod': {'request.dispatch': d}, + '/collection': {'request.dispatch': d}, + } + cherrypy.tree.mount(Root(), url, conf) + + + class Isolated: + def index(self): + return "made it!" + index.exposed = True + + cherrypy.tree.mount(Isolated(), "/isolated") + + class AnotherApp: + + exposed = True + + def GET(self): + return "milk" + + cherrypy.tree.mount(AnotherApp(), "/app", {'/': {'request.dispatch': d}}) + setup_server = staticmethod(setup_server) + + + def testObjectMapping(self): + for url in script_names: + prefix = self.script_name = url + + self.getPage('/') + self.assertBody('world') + + self.getPage("/dir1/myMethod") + self.assertBody("myMethod from dir1, path_info is:'/dir1/myMethod'") + + self.getPage("/this/method/does/not/exist") + self.assertBody("default:('this', 'method', 'does', 'not', 'exist')") + + self.getPage("/extra/too/much") + self.assertBody("('too', 'much')") + + self.getPage("/other") + self.assertBody('other') + + self.getPage("/notExposed") + self.assertBody("default:('notExposed',)") + + self.getPage("/dir1/dir2/") + self.assertBody('index for dir2, path is:/dir1/dir2/') + + # Test omitted trailing slash (should be redirected by default). + self.getPage("/dir1/dir2") + self.assertStatus(301) + self.assertHeader('Location', '%s/dir1/dir2/' % self.base()) + + # Test extra trailing slash (should be redirected if configured). + self.getPage("/dir1/myMethod/") + self.assertStatus(301) + self.assertHeader('Location', '%s/dir1/myMethod' % self.base()) + + # Test that default method must be exposed in order to match. + self.getPage("/dir1/dir2/dir3/dir4/index") + self.assertBody("default for dir1, param is:('dir2', 'dir3', 'dir4', 'index')") + + # Test *vpath when default() is defined but not index() + # This also tests HTTPRedirect with default. + self.getPage("/defnoindex") + self.assertStatus((302, 303)) + self.assertHeader('Location', '%s/contact' % self.base()) + self.getPage("/defnoindex/") + self.assertStatus((302, 303)) + self.assertHeader('Location', '%s/defnoindex/contact' % self.base()) + self.getPage("/defnoindex/page") + self.assertStatus((302, 303)) + self.assertHeader('Location', '%s/defnoindex/contact' % self.base()) + + self.getPage("/redirect") + self.assertStatus('302 Found') + self.assertHeader('Location', '%s/dir1/' % self.base()) + + if not getattr(cherrypy.server, "using_apache", False): + # Test that we can use URL's which aren't all valid Python identifiers + # This should also test the %XX-unquoting of URL's. + self.getPage("/Von%20B%fclow?ID=14") + self.assertBody("ID is 14") + + # Test that %2F in the path doesn't get unquoted too early; + # that is, it should not be used to separate path components. + # See ticket #393. + self.getPage("/page%2Fname") + self.assertBody("default:('page/name',)") + + self.getPage("/dir1/dir2/script_name") + self.assertBody(url) + self.getPage("/dir1/dir2/cherrypy_url") + self.assertBody("%s/extra" % self.base()) + + # Test that configs don't overwrite each other from diferent apps + self.getPage("/confvalue") + self.assertBody((url or "/").split("/")[-2]) + + self.script_name = "" + + # Test absoluteURI's in the Request-Line + self.getPage('http://%s:%s/' % (self.interface(), self.PORT)) + self.assertBody('world') + + self.getPage('http://%s:%s/abs/?service=http://192.168.0.1/x/y/z' % + (self.interface(), self.PORT)) + self.assertBody("default:('abs',)") + + self.getPage('/rel/?service=http://192.168.120.121:8000/x/y/z') + self.assertBody("default:('rel',)") + + # Test that the "isolated" app doesn't leak url's into the root app. + # If it did leak, Root.default() would answer with + # "default:('isolated', 'doesnt', 'exist')". + self.getPage("/isolated/") + self.assertStatus("200 OK") + self.assertBody("made it!") + self.getPage("/isolated/doesnt/exist") + self.assertStatus("404 Not Found") + + # Make sure /foobar maps to Root.foobar and not to the app + # mounted at /foo. See http://www.cherrypy.org/ticket/573 + self.getPage("/foobar") + self.assertBody("bar") + + def test_translate(self): + self.getPage("/translate_html") + self.assertStatus("200 OK") + self.assertBody("OK") + + self.getPage("/translate.html") + self.assertStatus("200 OK") + self.assertBody("OK") + + self.getPage("/translate-html") + self.assertStatus("200 OK") + self.assertBody("OK") + + def test_redir_using_url(self): + for url in script_names: + prefix = self.script_name = url + + # Test the absolute path to the parent (leading slash) + self.getPage('/redirect_via_url?path=./') + self.assertStatus(('302 Found', '303 See Other')) + self.assertHeader('Location', '%s/' % self.base()) + + # Test the relative path to the parent (no leading slash) + self.getPage('/redirect_via_url?path=./') + self.assertStatus(('302 Found', '303 See Other')) + self.assertHeader('Location', '%s/' % self.base()) + + # Test the absolute path to the parent (leading slash) + self.getPage('/redirect_via_url/?path=./') + self.assertStatus(('302 Found', '303 See Other')) + self.assertHeader('Location', '%s/' % self.base()) + + # Test the relative path to the parent (no leading slash) + self.getPage('/redirect_via_url/?path=./') + self.assertStatus(('302 Found', '303 See Other')) + self.assertHeader('Location', '%s/' % self.base()) + + def testPositionalParams(self): + self.getPage("/dir1/dir2/posparam/18/24/hut/hike") + self.assertBody("18/24/hut/hike") + + # intermediate index methods should not receive posparams; + # only the "final" index method should do so. + self.getPage("/dir1/dir2/5/3/sir") + self.assertBody("default for dir1, param is:('dir2', '5', '3', 'sir')") + + # test that extra positional args raises an 404 Not Found + # See http://www.cherrypy.org/ticket/733. + self.getPage("/dir1/dir2/script_name/extra/stuff") + self.assertStatus(404) + + def testExpose(self): + # Test the cherrypy.expose function/decorator + self.getPage("/exposing/base") + self.assertBody("expose works!") + + self.getPage("/exposing/1") + self.assertBody("expose works!") + + self.getPage("/exposing/2") + self.assertBody("expose works!") + + self.getPage("/exposingnew/base") + self.assertBody("expose works!") + + self.getPage("/exposingnew/1") + self.assertBody("expose works!") + + self.getPage("/exposingnew/2") + self.assertBody("expose works!") + + def testMethodDispatch(self): + self.getPage("/bymethod") + self.assertBody("['another']") + self.assertHeader('Allow', 'GET, HEAD, POST') + + self.getPage("/bymethod", method="HEAD") + self.assertBody("") + self.assertHeader('Allow', 'GET, HEAD, POST') + + self.getPage("/bymethod", method="POST", body="thing=one") + self.assertBody("") + self.assertHeader('Allow', 'GET, HEAD, POST') + + self.getPage("/bymethod") + self.assertBody("['another', u'one']") + self.assertHeader('Allow', 'GET, HEAD, POST') + + self.getPage("/bymethod", method="PUT") + self.assertErrorPage(405) + self.assertHeader('Allow', 'GET, HEAD, POST') + + # Test default with posparams + self.getPage("/collection/silly", method="POST") + self.getPage("/collection", method="GET") + self.assertBody("['a', 'bit', 'silly']") + + # Test custom dispatcher set on app root (see #737). + self.getPage("/app") + self.assertBody("milk") + + def testTreeMounting(self): + class Root(object): + def hello(self): + return "Hello world!" + hello.exposed = True + + # When mounting an application instance, + # we can't specify a different script name in the call to mount. + a = Application(Root(), '/somewhere') + self.assertRaises(ValueError, cherrypy.tree.mount, a, '/somewhereelse') + + # When mounting an application instance... + a = Application(Root(), '/somewhere') + # ...we MUST allow in identical script name in the call to mount... + cherrypy.tree.mount(a, '/somewhere') + self.getPage('/somewhere/hello') + self.assertStatus(200) + # ...and MUST allow a missing script_name. + del cherrypy.tree.apps['/somewhere'] + cherrypy.tree.mount(a) + self.getPage('/somewhere/hello') + self.assertStatus(200) + + # In addition, we MUST be able to create an Application using + # script_name == None for access to the wsgi_environ. + a = Application(Root(), script_name=None) + # However, this does not apply to tree.mount + self.assertRaises(TypeError, cherrypy.tree.mount, a, None) + diff --git a/cherrypy/test/test_proxy.py b/cherrypy/test/test_proxy.py new file mode 100755 index 0000000..2fbb619 --- /dev/null +++ b/cherrypy/test/test_proxy.py @@ -0,0 +1,129 @@ +import cherrypy +from cherrypy.test import helper + +script_names = ["", "/path/to/myapp"] + + +class ProxyTest(helper.CPWebCase): + + def setup_server(): + + # Set up site + cherrypy.config.update({ + 'tools.proxy.on': True, + 'tools.proxy.base': 'www.mydomain.test', + }) + + # Set up application + + class Root: + + def __init__(self, sn): + # Calculate a URL outside of any requests. + self.thisnewpage = cherrypy.url("/this/new/page", script_name=sn) + + def pageurl(self): + return self.thisnewpage + pageurl.exposed = True + + def index(self): + raise cherrypy.HTTPRedirect('dummy') + index.exposed = True + + def remoteip(self): + return cherrypy.request.remote.ip + remoteip.exposed = True + + def xhost(self): + raise cherrypy.HTTPRedirect('blah') + xhost.exposed = True + xhost._cp_config = {'tools.proxy.local': 'X-Host', + 'tools.trailing_slash.extra': True, + } + + def base(self): + return cherrypy.request.base + base.exposed = True + + def ssl(self): + return cherrypy.request.base + ssl.exposed = True + ssl._cp_config = {'tools.proxy.scheme': 'X-Forwarded-Ssl'} + + def newurl(self): + return ("Browse to <a href='%s'>this page</a>." + % cherrypy.url("/this/new/page")) + newurl.exposed = True + + for sn in script_names: + cherrypy.tree.mount(Root(sn), sn) + setup_server = staticmethod(setup_server) + + def testProxy(self): + self.getPage("/") + self.assertHeader('Location', + "%s://www.mydomain.test%s/dummy" % + (self.scheme, self.prefix())) + + # Test X-Forwarded-Host (Apache 1.3.33+ and Apache 2) + self.getPage("/", headers=[('X-Forwarded-Host', 'http://www.example.test')]) + self.assertHeader('Location', "http://www.example.test/dummy") + self.getPage("/", headers=[('X-Forwarded-Host', 'www.example.test')]) + self.assertHeader('Location', "%s://www.example.test/dummy" % self.scheme) + # Test multiple X-Forwarded-Host headers + self.getPage("/", headers=[ + ('X-Forwarded-Host', 'http://www.example.test, www.cherrypy.test'), + ]) + self.assertHeader('Location', "http://www.example.test/dummy") + + # Test X-Forwarded-For (Apache2) + self.getPage("/remoteip", + headers=[('X-Forwarded-For', '192.168.0.20')]) + self.assertBody("192.168.0.20") + self.getPage("/remoteip", + headers=[('X-Forwarded-For', '67.15.36.43, 192.168.0.20')]) + self.assertBody("192.168.0.20") + + # Test X-Host (lighttpd; see https://trac.lighttpd.net/trac/ticket/418) + self.getPage("/xhost", headers=[('X-Host', 'www.example.test')]) + self.assertHeader('Location', "%s://www.example.test/blah" % self.scheme) + + # Test X-Forwarded-Proto (lighttpd) + self.getPage("/base", headers=[('X-Forwarded-Proto', 'https')]) + self.assertBody("https://www.mydomain.test") + + # Test X-Forwarded-Ssl (webfaction?) + self.getPage("/ssl", headers=[('X-Forwarded-Ssl', 'on')]) + self.assertBody("https://www.mydomain.test") + + # Test cherrypy.url() + for sn in script_names: + # Test the value inside requests + self.getPage(sn + "/newurl") + self.assertBody("Browse to <a href='%s://www.mydomain.test" % self.scheme + + sn + "/this/new/page'>this page</a>.") + self.getPage(sn + "/newurl", headers=[('X-Forwarded-Host', + 'http://www.example.test')]) + self.assertBody("Browse to <a href='http://www.example.test" + + sn + "/this/new/page'>this page</a>.") + + # Test the value outside requests + port = "" + if self.scheme == "http" and self.PORT != 80: + port = ":%s" % self.PORT + elif self.scheme == "https" and self.PORT != 443: + port = ":%s" % self.PORT + host = self.HOST + if host in ('0.0.0.0', '::'): + import socket + host = socket.gethostname() + expected = ("%s://%s%s%s/this/new/page" + % (self.scheme, host, port, sn)) + self.getPage(sn + "/pageurl") + self.assertBody(expected) + + # Test trailing slash (see http://www.cherrypy.org/ticket/562). + self.getPage("/xhost/", headers=[('X-Host', 'www.example.test')]) + self.assertHeader('Location', "%s://www.example.test/xhost" + % self.scheme) + diff --git a/cherrypy/test/test_refleaks.py b/cherrypy/test/test_refleaks.py new file mode 100755 index 0000000..4df1f08 --- /dev/null +++ b/cherrypy/test/test_refleaks.py @@ -0,0 +1,119 @@ +"""Tests for refleaks.""" + +import gc +from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob +import threading + +import cherrypy +from cherrypy import _cprequest + + +data = object() + +def get_instances(cls): + return [x for x in gc.get_objects() if isinstance(x, cls)] + + +from cherrypy.test import helper + + +class ReferenceTests(helper.CPWebCase): + + def setup_server(): + + class Root: + def index(self, *args, **kwargs): + cherrypy.request.thing = data + return "Hello world!" + index.exposed = True + + def gc_stats(self): + output = ["Statistics:"] + + # Uncollectable garbage + + # gc_collect isn't perfectly synchronous, because it may + # break reference cycles that then take time to fully + # finalize. Call it twice and hope for the best. + gc.collect() + unreachable = gc.collect() + if unreachable: + output.append("\n%s unreachable objects:" % unreachable) + trash = {} + for x in gc.garbage: + trash[type(x)] = trash.get(type(x), 0) + 1 + trash = [(v, k) for k, v in trash.items()] + trash.sort() + for pair in trash: + output.append(" " + repr(pair)) + + # Request references + reqs = get_instances(_cprequest.Request) + lenreqs = len(reqs) + if lenreqs < 2: + output.append("\nMissing Request reference. Should be 1 in " + "this request thread and 1 in the main thread.") + elif lenreqs > 2: + output.append("\nToo many Request references (%r)." % lenreqs) + for req in reqs: + output.append("Referrers for %s:" % repr(req)) + for ref in gc.get_referrers(req): + if ref is not reqs: + output.append(" %s" % repr(ref)) + + # Response references + resps = get_instances(_cprequest.Response) + lenresps = len(resps) + if lenresps < 2: + output.append("\nMissing Response reference. Should be 1 in " + "this request thread and 1 in the main thread.") + elif lenresps > 2: + output.append("\nToo many Response references (%r)." % lenresps) + for resp in resps: + output.append("Referrers for %s:" % repr(resp)) + for ref in gc.get_referrers(resp): + if ref is not resps: + output.append(" %s" % repr(ref)) + + return "\n".join(output) + gc_stats.exposed = True + + cherrypy.tree.mount(Root()) + setup_server = staticmethod(setup_server) + + + def test_threadlocal_garbage(self): + success = [] + + def getpage(): + host = '%s:%s' % (self.interface(), self.PORT) + if self.scheme == 'https': + c = HTTPSConnection(host) + else: + c = HTTPConnection(host) + try: + c.putrequest('GET', '/') + c.endheaders() + response = c.getresponse() + body = response.read() + self.assertEqual(response.status, 200) + self.assertEqual(body, ntob("Hello world!")) + finally: + c.close() + success.append(True) + + ITERATIONS = 25 + ts = [] + for _ in range(ITERATIONS): + t = threading.Thread(target=getpage) + ts.append(t) + t.start() + + for t in ts: + t.join() + + self.assertEqual(len(success), ITERATIONS) + + self.getPage("/gc_stats") + self.assertBody("Statistics:") + diff --git a/cherrypy/test/test_request_obj.py b/cherrypy/test/test_request_obj.py new file mode 100755 index 0000000..91ee4fd --- /dev/null +++ b/cherrypy/test/test_request_obj.py @@ -0,0 +1,722 @@ +"""Basic tests for the cherrypy.Request object.""" + +import os +localDir = os.path.dirname(__file__) +import sys +import types +from cherrypy._cpcompat import IncompleteRead, ntob, unicodestr + +import cherrypy +from cherrypy import _cptools, tools +from cherrypy.lib import httputil + +defined_http_methods = ("OPTIONS", "GET", "HEAD", "POST", "PUT", "DELETE", + "TRACE", "PROPFIND") + + +# Client-side code # + +from cherrypy.test import helper + +class RequestObjectTests(helper.CPWebCase): + + def setup_server(): + class Root: + + def index(self): + return "hello" + index.exposed = True + + def scheme(self): + return cherrypy.request.scheme + scheme.exposed = True + + root = Root() + + + class TestType(type): + """Metaclass which automatically exposes all functions in each subclass, + and adds an instance of the subclass as an attribute of root. + """ + def __init__(cls, name, bases, dct): + type.__init__(cls, name, bases, dct) + for value in dct.values(): + if isinstance(value, types.FunctionType): + value.exposed = True + setattr(root, name.lower(), cls()) + class Test(object): + __metaclass__ = TestType + + + class Params(Test): + + def index(self, thing): + return repr(thing) + + def ismap(self, x, y): + return "Coordinates: %s, %s" % (x, y) + + def default(self, *args, **kwargs): + return "args: %s kwargs: %s" % (args, kwargs) + default._cp_config = {'request.query_string_encoding': 'latin1'} + + + class ParamErrorsCallable(object): + exposed = True + def __call__(self): + return "data" + + class ParamErrors(Test): + + def one_positional(self, param1): + return "data" + one_positional.exposed = True + + def one_positional_args(self, param1, *args): + return "data" + one_positional_args.exposed = True + + def one_positional_args_kwargs(self, param1, *args, **kwargs): + return "data" + one_positional_args_kwargs.exposed = True + + def one_positional_kwargs(self, param1, **kwargs): + return "data" + one_positional_kwargs.exposed = True + + def no_positional(self): + return "data" + no_positional.exposed = True + + def no_positional_args(self, *args): + return "data" + no_positional_args.exposed = True + + def no_positional_args_kwargs(self, *args, **kwargs): + return "data" + no_positional_args_kwargs.exposed = True + + def no_positional_kwargs(self, **kwargs): + return "data" + no_positional_kwargs.exposed = True + + callable_object = ParamErrorsCallable() + + def raise_type_error(self, **kwargs): + raise TypeError("Client Error") + raise_type_error.exposed = True + + def raise_type_error_with_default_param(self, x, y=None): + return '%d' % 'a' # throw an exception + raise_type_error_with_default_param.exposed = True + + def callable_error_page(status, **kwargs): + return "Error %s - Well, I'm very sorry but you haven't paid!" % status + + + class Error(Test): + + _cp_config = {'tools.log_tracebacks.on': True, + } + + def reason_phrase(self): + raise cherrypy.HTTPError("410 Gone fishin'") + + def custom(self, err='404'): + raise cherrypy.HTTPError(int(err), "No, <b>really</b>, not found!") + custom._cp_config = {'error_page.404': os.path.join(localDir, "static/index.html"), + 'error_page.401': callable_error_page, + } + + def custom_default(self): + return 1 + 'a' # raise an unexpected error + custom_default._cp_config = {'error_page.default': callable_error_page} + + def noexist(self): + raise cherrypy.HTTPError(404, "No, <b>really</b>, not found!") + noexist._cp_config = {'error_page.404': "nonexistent.html"} + + def page_method(self): + raise ValueError() + + def page_yield(self): + yield "howdy" + raise ValueError() + + def page_streamed(self): + yield "word up" + raise ValueError() + yield "very oops" + page_streamed._cp_config = {"response.stream": True} + + def cause_err_in_finalize(self): + # Since status must start with an int, this should error. + cherrypy.response.status = "ZOO OK" + cause_err_in_finalize._cp_config = {'request.show_tracebacks': False} + + def rethrow(self): + """Test that an error raised here will be thrown out to the server.""" + raise ValueError() + rethrow._cp_config = {'request.throw_errors': True} + + + class Expect(Test): + + def expectation_failed(self): + expect = cherrypy.request.headers.elements("Expect") + if expect and expect[0].value != '100-continue': + raise cherrypy.HTTPError(400) + raise cherrypy.HTTPError(417, 'Expectation Failed') + + class Headers(Test): + + def default(self, headername): + """Spit back out the value for the requested header.""" + return cherrypy.request.headers[headername] + + def doubledheaders(self): + # From http://www.cherrypy.org/ticket/165: + # "header field names should not be case sensitive sayes the rfc. + # if i set a headerfield in complete lowercase i end up with two + # header fields, one in lowercase, the other in mixed-case." + + # Set the most common headers + hMap = cherrypy.response.headers + hMap['content-type'] = "text/html" + hMap['content-length'] = 18 + hMap['server'] = 'CherryPy headertest' + hMap['location'] = ('%s://%s:%s/headers/' + % (cherrypy.request.local.ip, + cherrypy.request.local.port, + cherrypy.request.scheme)) + + # Set a rare header for fun + hMap['Expires'] = 'Thu, 01 Dec 2194 16:00:00 GMT' + + return "double header test" + + def ifmatch(self): + val = cherrypy.request.headers['If-Match'] + assert isinstance(val, unicodestr) + cherrypy.response.headers['ETag'] = val + return val + + + class HeaderElements(Test): + + def get_elements(self, headername): + e = cherrypy.request.headers.elements(headername) + return "\n".join([unicodestr(x) for x in e]) + + + class Method(Test): + + def index(self): + m = cherrypy.request.method + if m in defined_http_methods or m == "CONNECT": + return m + + if m == "LINK": + raise cherrypy.HTTPError(405) + else: + raise cherrypy.HTTPError(501) + + def parameterized(self, data): + return data + + def request_body(self): + # This should be a file object (temp file), + # which CP will just pipe back out if we tell it to. + return cherrypy.request.body + + def reachable(self): + return "success" + + class Divorce: + """HTTP Method handlers shouldn't collide with normal method names. + For example, a GET-handler shouldn't collide with a method named 'get'. + + If you build HTTP method dispatching into CherryPy, rewrite this class + to use your new dispatch mechanism and make sure that: + "GET /divorce HTTP/1.1" maps to divorce.index() and + "GET /divorce/get?ID=13 HTTP/1.1" maps to divorce.get() + """ + + documents = {} + + def index(self): + yield "<h1>Choose your document</h1>\n" + yield "<ul>\n" + for id, contents in self.documents.items(): + yield (" <li><a href='/divorce/get?ID=%s'>%s</a>: %s</li>\n" + % (id, id, contents)) + yield "</ul>" + index.exposed = True + + def get(self, ID): + return ("Divorce document %s: %s" % + (ID, self.documents.get(ID, "empty"))) + get.exposed = True + + root.divorce = Divorce() + + + class ThreadLocal(Test): + + def index(self): + existing = repr(getattr(cherrypy.request, "asdf", None)) + cherrypy.request.asdf = "rassfrassin" + return existing + + appconf = { + '/method': {'request.methods_with_bodies': ("POST", "PUT", "PROPFIND")}, + } + cherrypy.tree.mount(root, config=appconf) + setup_server = staticmethod(setup_server) + + def test_scheme(self): + self.getPage("/scheme") + self.assertBody(self.scheme) + + def testParams(self): + self.getPage("/params/?thing=a") + self.assertBody("u'a'") + + self.getPage("/params/?thing=a&thing=b&thing=c") + self.assertBody("[u'a', u'b', u'c']") + + # Test friendly error message when given params are not accepted. + cherrypy.config.update({"request.show_mismatched_params": True}) + self.getPage("/params/?notathing=meeting") + self.assertInBody("Missing parameters: thing") + self.getPage("/params/?thing=meeting¬athing=meeting") + self.assertInBody("Unexpected query string parameters: notathing") + + # Test ability to turn off friendly error messages + cherrypy.config.update({"request.show_mismatched_params": False}) + self.getPage("/params/?notathing=meeting") + self.assertInBody("Not Found") + self.getPage("/params/?thing=meeting¬athing=meeting") + self.assertInBody("Not Found") + + # Test "% HEX HEX"-encoded URL, param keys, and values + self.getPage("/params/%d4%20%e3/cheese?Gruy%E8re=Bulgn%e9ville") + self.assertBody(r"args: ('\xd4 \xe3', 'cheese') " + r"kwargs: {'Gruy\xe8re': u'Bulgn\xe9ville'}") + + # Make sure that encoded = and & get parsed correctly + self.getPage("/params/code?url=http%3A//cherrypy.org/index%3Fa%3D1%26b%3D2") + self.assertBody(r"args: ('code',) " + r"kwargs: {'url': u'http://cherrypy.org/index?a=1&b=2'}") + + # Test coordinates sent by <img ismap> + self.getPage("/params/ismap?223,114") + self.assertBody("Coordinates: 223, 114") + + # Test "name[key]" dict-like params + self.getPage("/params/dictlike?a[1]=1&a[2]=2&b=foo&b[bar]=baz") + self.assertBody( + "args: ('dictlike',) " + "kwargs: {'a[1]': u'1', 'b[bar]': u'baz', 'b': u'foo', 'a[2]': u'2'}") + + def testParamErrors(self): + + # test that all of the handlers work when given + # the correct parameters in order to ensure that the + # errors below aren't coming from some other source. + for uri in ( + '/paramerrors/one_positional?param1=foo', + '/paramerrors/one_positional_args?param1=foo', + '/paramerrors/one_positional_args/foo', + '/paramerrors/one_positional_args/foo/bar/baz', + '/paramerrors/one_positional_args_kwargs?param1=foo¶m2=bar', + '/paramerrors/one_positional_args_kwargs/foo?param2=bar¶m3=baz', + '/paramerrors/one_positional_args_kwargs/foo/bar/baz?param2=bar¶m3=baz', + '/paramerrors/one_positional_kwargs?param1=foo¶m2=bar¶m3=baz', + '/paramerrors/one_positional_kwargs/foo?param4=foo¶m2=bar¶m3=baz', + '/paramerrors/no_positional', + '/paramerrors/no_positional_args/foo', + '/paramerrors/no_positional_args/foo/bar/baz', + '/paramerrors/no_positional_args_kwargs?param1=foo¶m2=bar', + '/paramerrors/no_positional_args_kwargs/foo?param2=bar', + '/paramerrors/no_positional_args_kwargs/foo/bar/baz?param2=bar¶m3=baz', + '/paramerrors/no_positional_kwargs?param1=foo¶m2=bar', + '/paramerrors/callable_object', + ): + self.getPage(uri) + self.assertStatus(200) + + # query string parameters are part of the URI, so if they are wrong + # for a particular handler, the status MUST be a 404. + error_msgs = [ + 'Missing parameters', + 'Nothing matches the given URI', + 'Multiple values for parameters', + 'Unexpected query string parameters', + 'Unexpected body parameters', + ] + for uri, msg in ( + ('/paramerrors/one_positional', error_msgs[0]), + ('/paramerrors/one_positional?foo=foo', error_msgs[0]), + ('/paramerrors/one_positional/foo/bar/baz', error_msgs[1]), + ('/paramerrors/one_positional/foo?param1=foo', error_msgs[2]), + ('/paramerrors/one_positional/foo?param1=foo¶m2=foo', error_msgs[2]), + ('/paramerrors/one_positional_args/foo?param1=foo¶m2=foo', error_msgs[2]), + ('/paramerrors/one_positional_args/foo/bar/baz?param2=foo', error_msgs[3]), + ('/paramerrors/one_positional_args_kwargs/foo/bar/baz?param1=bar¶m3=baz', error_msgs[2]), + ('/paramerrors/one_positional_kwargs/foo?param1=foo¶m2=bar¶m3=baz', error_msgs[2]), + ('/paramerrors/no_positional/boo', error_msgs[1]), + ('/paramerrors/no_positional?param1=foo', error_msgs[3]), + ('/paramerrors/no_positional_args/boo?param1=foo', error_msgs[3]), + ('/paramerrors/no_positional_kwargs/boo?param1=foo', error_msgs[1]), + ('/paramerrors/callable_object?param1=foo', error_msgs[3]), + ('/paramerrors/callable_object/boo', error_msgs[1]), + ): + for show_mismatched_params in (True, False): + cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) + self.getPage(uri) + self.assertStatus(404) + if show_mismatched_params: + self.assertInBody(msg) + else: + self.assertInBody("Not Found") + + # if body parameters are wrong, a 400 must be returned. + for uri, body, msg in ( + ('/paramerrors/one_positional/foo', 'param1=foo', error_msgs[2]), + ('/paramerrors/one_positional/foo', 'param1=foo¶m2=foo', error_msgs[2]), + ('/paramerrors/one_positional_args/foo', 'param1=foo¶m2=foo', error_msgs[2]), + ('/paramerrors/one_positional_args/foo/bar/baz', 'param2=foo', error_msgs[4]), + ('/paramerrors/one_positional_args_kwargs/foo/bar/baz', 'param1=bar¶m3=baz', error_msgs[2]), + ('/paramerrors/one_positional_kwargs/foo', 'param1=foo¶m2=bar¶m3=baz', error_msgs[2]), + ('/paramerrors/no_positional', 'param1=foo', error_msgs[4]), + ('/paramerrors/no_positional_args/boo', 'param1=foo', error_msgs[4]), + ('/paramerrors/callable_object', 'param1=foo', error_msgs[4]), + ): + for show_mismatched_params in (True, False): + cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) + self.getPage(uri, method='POST', body=body) + self.assertStatus(400) + if show_mismatched_params: + self.assertInBody(msg) + else: + self.assertInBody("Bad Request") + + + # even if body parameters are wrong, if we get the uri wrong, then + # it's a 404 + for uri, body, msg in ( + ('/paramerrors/one_positional?param2=foo', 'param1=foo', error_msgs[3]), + ('/paramerrors/one_positional/foo/bar', 'param2=foo', error_msgs[1]), + ('/paramerrors/one_positional_args/foo/bar?param2=foo', 'param3=foo', error_msgs[3]), + ('/paramerrors/one_positional_kwargs/foo/bar', 'param2=bar¶m3=baz', error_msgs[1]), + ('/paramerrors/no_positional?param1=foo', 'param2=foo', error_msgs[3]), + ('/paramerrors/no_positional_args/boo?param2=foo', 'param1=foo', error_msgs[3]), + ('/paramerrors/callable_object?param2=bar', 'param1=foo', error_msgs[3]), + ): + for show_mismatched_params in (True, False): + cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) + self.getPage(uri, method='POST', body=body) + self.assertStatus(404) + if show_mismatched_params: + self.assertInBody(msg) + else: + self.assertInBody("Not Found") + + # In the case that a handler raises a TypeError we should + # let that type error through. + for uri in ( + '/paramerrors/raise_type_error', + '/paramerrors/raise_type_error_with_default_param?x=0', + '/paramerrors/raise_type_error_with_default_param?x=0&y=0', + ): + self.getPage(uri, method='GET') + self.assertStatus(500) + self.assertTrue('Client Error', self.body) + + def testErrorHandling(self): + self.getPage("/error/missing") + self.assertStatus(404) + self.assertErrorPage(404, "The path '/error/missing' was not found.") + + ignore = helper.webtest.ignored_exceptions + ignore.append(ValueError) + try: + valerr = '\n raise ValueError()\nValueError' + self.getPage("/error/page_method") + self.assertErrorPage(500, pattern=valerr) + + self.getPage("/error/page_yield") + self.assertErrorPage(500, pattern=valerr) + + if (cherrypy.server.protocol_version == "HTTP/1.0" or + getattr(cherrypy.server, "using_apache", False)): + self.getPage("/error/page_streamed") + # Because this error is raised after the response body has + # started, the status should not change to an error status. + self.assertStatus(200) + self.assertBody("word up") + else: + # Under HTTP/1.1, the chunked transfer-coding is used. + # The HTTP client will choke when the output is incomplete. + self.assertRaises((ValueError, IncompleteRead), self.getPage, + "/error/page_streamed") + + # No traceback should be present + self.getPage("/error/cause_err_in_finalize") + msg = "Illegal response status from server ('ZOO' is non-numeric)." + self.assertErrorPage(500, msg, None) + finally: + ignore.pop() + + # Test HTTPError with a reason-phrase in the status arg. + self.getPage('/error/reason_phrase') + self.assertStatus("410 Gone fishin'") + + # Test custom error page for a specific error. + self.getPage("/error/custom") + self.assertStatus(404) + self.assertBody("Hello, world\r\n" + (" " * 499)) + + # Test custom error page for a specific error. + self.getPage("/error/custom?err=401") + self.assertStatus(401) + self.assertBody("Error 401 Unauthorized - Well, I'm very sorry but you haven't paid!") + + # Test default custom error page. + self.getPage("/error/custom_default") + self.assertStatus(500) + self.assertBody("Error 500 Internal Server Error - Well, I'm very sorry but you haven't paid!".ljust(513)) + + # Test error in custom error page (ticket #305). + # Note that the message is escaped for HTML (ticket #310). + self.getPage("/error/noexist") + self.assertStatus(404) + msg = ("No, <b>really</b>, not found!<br />" + "In addition, the custom error page failed:\n<br />" + "IOError: [Errno 2] No such file or directory: 'nonexistent.html'") + self.assertInBody(msg) + + if getattr(cherrypy.server, "using_apache", False): + pass + else: + # Test throw_errors (ticket #186). + self.getPage("/error/rethrow") + self.assertInBody("raise ValueError()") + + def testExpect(self): + e = ('Expect', '100-continue') + self.getPage("/headerelements/get_elements?headername=Expect", [e]) + self.assertBody('100-continue') + + self.getPage("/expect/expectation_failed", [e]) + self.assertStatus(417) + + def testHeaderElements(self): + # Accept-* header elements should be sorted, with most preferred first. + h = [('Accept', 'audio/*; q=0.2, audio/basic')] + self.getPage("/headerelements/get_elements?headername=Accept", h) + self.assertStatus(200) + self.assertBody("audio/basic\n" + "audio/*;q=0.2") + + h = [('Accept', 'text/plain; q=0.5, text/html, text/x-dvi; q=0.8, text/x-c')] + self.getPage("/headerelements/get_elements?headername=Accept", h) + self.assertStatus(200) + self.assertBody("text/x-c\n" + "text/html\n" + "text/x-dvi;q=0.8\n" + "text/plain;q=0.5") + + # Test that more specific media ranges get priority. + h = [('Accept', 'text/*, text/html, text/html;level=1, */*')] + self.getPage("/headerelements/get_elements?headername=Accept", h) + self.assertStatus(200) + self.assertBody("text/html;level=1\n" + "text/html\n" + "text/*\n" + "*/*") + + # Test Accept-Charset + h = [('Accept-Charset', 'iso-8859-5, unicode-1-1;q=0.8')] + self.getPage("/headerelements/get_elements?headername=Accept-Charset", h) + self.assertStatus("200 OK") + self.assertBody("iso-8859-5\n" + "unicode-1-1;q=0.8") + + # Test Accept-Encoding + h = [('Accept-Encoding', 'gzip;q=1.0, identity; q=0.5, *;q=0')] + self.getPage("/headerelements/get_elements?headername=Accept-Encoding", h) + self.assertStatus("200 OK") + self.assertBody("gzip;q=1.0\n" + "identity;q=0.5\n" + "*;q=0") + + # Test Accept-Language + h = [('Accept-Language', 'da, en-gb;q=0.8, en;q=0.7')] + self.getPage("/headerelements/get_elements?headername=Accept-Language", h) + self.assertStatus("200 OK") + self.assertBody("da\n" + "en-gb;q=0.8\n" + "en;q=0.7") + + # Test malformed header parsing. See http://www.cherrypy.org/ticket/763. + self.getPage("/headerelements/get_elements?headername=Content-Type", + # Note the illegal trailing ";" + headers=[('Content-Type', 'text/html; charset=utf-8;')]) + self.assertStatus(200) + self.assertBody("text/html;charset=utf-8") + + def test_repeated_headers(self): + # Test that two request headers are collapsed into one. + # See http://www.cherrypy.org/ticket/542. + self.getPage("/headers/Accept-Charset", + headers=[("Accept-Charset", "iso-8859-5"), + ("Accept-Charset", "unicode-1-1;q=0.8")]) + self.assertBody("iso-8859-5, unicode-1-1;q=0.8") + + # Tests that each header only appears once, regardless of case. + self.getPage("/headers/doubledheaders") + self.assertBody("double header test") + hnames = [name.title() for name, val in self.headers] + for key in ['Content-Length', 'Content-Type', 'Date', + 'Expires', 'Location', 'Server']: + self.assertEqual(hnames.count(key), 1, self.headers) + + def test_encoded_headers(self): + # First, make sure the innards work like expected. + self.assertEqual(httputil.decode_TEXT(u"=?utf-8?q?f=C3=BCr?="), u"f\xfcr") + + if cherrypy.server.protocol_version == "HTTP/1.1": + # Test RFC-2047-encoded request and response header values + u = u'\u212bngstr\xf6m' + c = u"=E2=84=ABngstr=C3=B6m" + self.getPage("/headers/ifmatch", [('If-Match', u'=?utf-8?q?%s?=' % c)]) + # The body should be utf-8 encoded. + self.assertBody("\xe2\x84\xabngstr\xc3\xb6m") + # But the Etag header should be RFC-2047 encoded (binary) + self.assertHeader("ETag", u'=?utf-8?b?4oSrbmdzdHLDtm0=?=') + + # Test a *LONG* RFC-2047-encoded request and response header value + self.getPage("/headers/ifmatch", + [('If-Match', u'=?utf-8?q?%s?=' % (c * 10))]) + self.assertBody("\xe2\x84\xabngstr\xc3\xb6m" * 10) + # Note: this is different output for Python3, but it decodes fine. + etag = self.assertHeader("ETag", + '=?utf-8?b?4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' + '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' + '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' + '4oSrbmdzdHLDtm0=?=') + self.assertEqual(httputil.decode_TEXT(etag), u * 10) + + def test_header_presence(self): + # If we don't pass a Content-Type header, it should not be present + # in cherrypy.request.headers + self.getPage("/headers/Content-Type", + headers=[]) + self.assertStatus(500) + + # If Content-Type is present in the request, it should be present in + # cherrypy.request.headers + self.getPage("/headers/Content-Type", + headers=[("Content-type", "application/json")]) + self.assertBody("application/json") + + def test_basic_HTTPMethods(self): + helper.webtest.methods_with_bodies = ("POST", "PUT", "PROPFIND") + + # Test that all defined HTTP methods work. + for m in defined_http_methods: + self.getPage("/method/", method=m) + + # HEAD requests should not return any body. + if m == "HEAD": + self.assertBody("") + elif m == "TRACE": + # Some HTTP servers (like modpy) have their own TRACE support + self.assertEqual(self.body[:5], ntob("TRACE")) + else: + self.assertBody(m) + + # Request a PUT method with a form-urlencoded body + self.getPage("/method/parameterized", method="PUT", + body="data=on+top+of+other+things") + self.assertBody("on top of other things") + + # Request a PUT method with a file body + b = "one thing on top of another" + h = [("Content-Type", "text/plain"), + ("Content-Length", str(len(b)))] + self.getPage("/method/request_body", headers=h, method="PUT", body=b) + self.assertStatus(200) + self.assertBody(b) + + # Request a PUT method with a file body but no Content-Type. + # See http://www.cherrypy.org/ticket/790. + b = ntob("one thing on top of another") + self.persistent = True + try: + conn = self.HTTP_CONN + conn.putrequest("PUT", "/method/request_body", skip_host=True) + conn.putheader("Host", self.HOST) + conn.putheader('Content-Length', str(len(b))) + conn.endheaders() + conn.send(b) + response = conn.response_class(conn.sock, method="PUT") + response.begin() + self.assertEqual(response.status, 200) + self.body = response.read() + self.assertBody(b) + finally: + self.persistent = False + + # Request a PUT method with no body whatsoever (not an empty one). + # See http://www.cherrypy.org/ticket/650. + # Provide a C-T or webtest will provide one (and a C-L) for us. + h = [("Content-Type", "text/plain")] + self.getPage("/method/reachable", headers=h, method="PUT") + self.assertStatus(411) + + # Request a custom method with a request body + b = ('<?xml version="1.0" encoding="utf-8" ?>\n\n' + '<propfind xmlns="DAV:"><prop><getlastmodified/>' + '</prop></propfind>') + h = [('Content-Type', 'text/xml'), + ('Content-Length', str(len(b)))] + self.getPage("/method/request_body", headers=h, method="PROPFIND", body=b) + self.assertStatus(200) + self.assertBody(b) + + # Request a disallowed method + self.getPage("/method/", method="LINK") + self.assertStatus(405) + + # Request an unknown method + self.getPage("/method/", method="SEARCH") + self.assertStatus(501) + + # For method dispatchers: make sure that an HTTP method doesn't + # collide with a virtual path atom. If you build HTTP-method + # dispatching into the core, rewrite these handlers to use + # your dispatch idioms. + self.getPage("/divorce/get?ID=13") + self.assertBody('Divorce document 13: empty') + self.assertStatus(200) + self.getPage("/divorce/", method="GET") + self.assertBody('<h1>Choose your document</h1>\n<ul>\n</ul>') + self.assertStatus(200) + + def test_CONNECT_method(self): + if getattr(cherrypy.server, "using_apache", False): + return self.skip("skipped due to known Apache differences... ") + + self.getPage("/method/", method="CONNECT") + self.assertBody("CONNECT") + + def testEmptyThreadlocals(self): + results = [] + for x in range(20): + self.getPage("/threadlocal/") + results.append(self.body) + self.assertEqual(results, [ntob("None")] * 20) + diff --git a/cherrypy/test/test_routes.py b/cherrypy/test/test_routes.py new file mode 100755 index 0000000..a8062f8 --- /dev/null +++ b/cherrypy/test/test_routes.py @@ -0,0 +1,69 @@ +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + +import cherrypy + +from cherrypy.test import helper +import nose + +class RoutesDispatchTest(helper.CPWebCase): + + def setup_server(): + + try: + import routes + except ImportError: + raise nose.SkipTest('Install routes to test RoutesDispatcher code') + + class Dummy: + def index(self): + return "I said good day!" + + class City: + + def __init__(self, name): + self.name = name + self.population = 10000 + + def index(self, **kwargs): + return "Welcome to %s, pop. %s" % (self.name, self.population) + index._cp_config = {'tools.response_headers.on': True, + 'tools.response_headers.headers': [('Content-Language', 'en-GB')]} + + def update(self, **kwargs): + self.population = kwargs['pop'] + return "OK" + + d = cherrypy.dispatch.RoutesDispatcher() + d.connect(action='index', name='hounslow', route='/hounslow', + controller=City('Hounslow')) + d.connect(name='surbiton', route='/surbiton', controller=City('Surbiton'), + action='index', conditions=dict(method=['GET'])) + d.mapper.connect('/surbiton', controller='surbiton', + action='update', conditions=dict(method=['POST'])) + d.connect('main', ':action', controller=Dummy()) + + conf = {'/': {'request.dispatch': d}} + cherrypy.tree.mount(root=None, config=conf) + setup_server = staticmethod(setup_server) + + def test_Routes_Dispatch(self): + self.getPage("/hounslow") + self.assertStatus("200 OK") + self.assertBody("Welcome to Hounslow, pop. 10000") + + self.getPage("/foo") + self.assertStatus("404 Not Found") + + self.getPage("/surbiton") + self.assertStatus("200 OK") + self.assertBody("Welcome to Surbiton, pop. 10000") + + self.getPage("/surbiton", method="POST", body="pop=1327") + self.assertStatus("200 OK") + self.assertBody("OK") + self.getPage("/surbiton") + self.assertStatus("200 OK") + self.assertHeader("Content-Language", "en-GB") + self.assertBody("Welcome to Surbiton, pop. 1327") + diff --git a/cherrypy/test/test_session.py b/cherrypy/test/test_session.py new file mode 100755 index 0000000..874023e --- /dev/null +++ b/cherrypy/test/test_session.py @@ -0,0 +1,464 @@ +import os +localDir = os.path.dirname(__file__) +import sys +import threading +import time + +import cherrypy +from cherrypy._cpcompat import copykeys, HTTPConnection, HTTPSConnection +from cherrypy.lib import sessions +from cherrypy.lib.httputil import response_codes + +def http_methods_allowed(methods=['GET', 'HEAD']): + method = cherrypy.request.method.upper() + if method not in methods: + cherrypy.response.headers['Allow'] = ", ".join(methods) + raise cherrypy.HTTPError(405) + +cherrypy.tools.allow = cherrypy.Tool('on_start_resource', http_methods_allowed) + + +def setup_server(): + + class Root: + + _cp_config = {'tools.sessions.on': True, + 'tools.sessions.storage_type' : 'ram', + 'tools.sessions.storage_path' : localDir, + 'tools.sessions.timeout': (1.0 / 60), + 'tools.sessions.clean_freq': (1.0 / 60), + } + + def clear(self): + cherrypy.session.cache.clear() + clear.exposed = True + + def data(self): + cherrypy.session['aha'] = 'foo' + return repr(cherrypy.session._data) + data.exposed = True + + def testGen(self): + counter = cherrypy.session.get('counter', 0) + 1 + cherrypy.session['counter'] = counter + yield str(counter) + testGen.exposed = True + + def testStr(self): + counter = cherrypy.session.get('counter', 0) + 1 + cherrypy.session['counter'] = counter + return str(counter) + testStr.exposed = True + + def setsessiontype(self, newtype): + self.__class__._cp_config.update({'tools.sessions.storage_type': newtype}) + if hasattr(cherrypy, "session"): + del cherrypy.session + cls = getattr(sessions, newtype.title() + 'Session') + if cls.clean_thread: + cls.clean_thread.stop() + cls.clean_thread.unsubscribe() + del cls.clean_thread + setsessiontype.exposed = True + setsessiontype._cp_config = {'tools.sessions.on': False} + + def index(self): + sess = cherrypy.session + c = sess.get('counter', 0) + 1 + time.sleep(0.01) + sess['counter'] = c + return str(c) + index.exposed = True + + def keyin(self, key): + return str(key in cherrypy.session) + keyin.exposed = True + + def delete(self): + cherrypy.session.delete() + sessions.expire() + return "done" + delete.exposed = True + + def delkey(self, key): + del cherrypy.session[key] + return "OK" + delkey.exposed = True + + def blah(self): + return self._cp_config['tools.sessions.storage_type'] + blah.exposed = True + + def iredir(self): + raise cherrypy.InternalRedirect('/blah') + iredir.exposed = True + + def restricted(self): + return cherrypy.request.method + restricted.exposed = True + restricted._cp_config = {'tools.allow.on': True, + 'tools.allow.methods': ['GET']} + + def regen(self): + cherrypy.tools.sessions.regenerate() + return "logged in" + regen.exposed = True + + def length(self): + return str(len(cherrypy.session)) + length.exposed = True + + def session_cookie(self): + # Must load() to start the clean thread. + cherrypy.session.load() + return cherrypy.session.id + session_cookie.exposed = True + session_cookie._cp_config = { + 'tools.sessions.path': '/session_cookie', + 'tools.sessions.name': 'temp', + 'tools.sessions.persistent': False} + + cherrypy.tree.mount(Root()) + + +from cherrypy.test import helper + +class SessionTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def tearDown(self): + # Clean up sessions. + for fname in os.listdir(localDir): + if fname.startswith(sessions.FileSession.SESSION_PREFIX): + os.unlink(os.path.join(localDir, fname)) + + def test_0_Session(self): + self.getPage('/setsessiontype/ram') + self.getPage('/clear') + + # Test that a normal request gets the same id in the cookies. + # Note: this wouldn't work if /data didn't load the session. + self.getPage('/data') + self.assertBody("{'aha': 'foo'}") + c = self.cookies[0] + self.getPage('/data', self.cookies) + self.assertEqual(self.cookies[0], c) + + self.getPage('/testStr') + self.assertBody('1') + cookie_parts = dict([p.strip().split('=') + for p in self.cookies[0][1].split(";")]) + # Assert there is an 'expires' param + self.assertEqual(set(cookie_parts.keys()), + set(['session_id', 'expires', 'Path'])) + self.getPage('/testGen', self.cookies) + self.assertBody('2') + self.getPage('/testStr', self.cookies) + self.assertBody('3') + self.getPage('/data', self.cookies) + self.assertBody("{'aha': 'foo', 'counter': 3}") + self.getPage('/length', self.cookies) + self.assertBody('2') + self.getPage('/delkey?key=counter', self.cookies) + self.assertStatus(200) + + self.getPage('/setsessiontype/file') + self.getPage('/testStr') + self.assertBody('1') + self.getPage('/testGen', self.cookies) + self.assertBody('2') + self.getPage('/testStr', self.cookies) + self.assertBody('3') + self.getPage('/delkey?key=counter', self.cookies) + self.assertStatus(200) + + # Wait for the session.timeout (1 second) + time.sleep(2) + self.getPage('/') + self.assertBody('1') + self.getPage('/length', self.cookies) + self.assertBody('1') + + # Test session __contains__ + self.getPage('/keyin?key=counter', self.cookies) + self.assertBody("True") + cookieset1 = self.cookies + + # Make a new session and test __len__ again + self.getPage('/') + self.getPage('/length', self.cookies) + self.assertBody('2') + + # Test session delete + self.getPage('/delete', self.cookies) + self.assertBody("done") + self.getPage('/delete', cookieset1) + self.assertBody("done") + f = lambda: [x for x in os.listdir(localDir) if x.startswith('session-')] + self.assertEqual(f(), []) + + # Wait for the cleanup thread to delete remaining session files + self.getPage('/') + f = lambda: [x for x in os.listdir(localDir) if x.startswith('session-')] + self.assertNotEqual(f(), []) + time.sleep(2) + self.assertEqual(f(), []) + + def test_1_Ram_Concurrency(self): + self.getPage('/setsessiontype/ram') + self._test_Concurrency() + + def test_2_File_Concurrency(self): + self.getPage('/setsessiontype/file') + self._test_Concurrency() + + def _test_Concurrency(self): + client_thread_count = 5 + request_count = 30 + + # Get initial cookie + self.getPage("/") + self.assertBody("1") + cookies = self.cookies + + data_dict = {} + errors = [] + + def request(index): + if self.scheme == 'https': + c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) + else: + c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) + for i in range(request_count): + c.putrequest('GET', '/') + for k, v in cookies: + c.putheader(k, v) + c.endheaders() + response = c.getresponse() + body = response.read() + if response.status != 200 or not body.isdigit(): + errors.append((response.status, body)) + else: + data_dict[index] = max(data_dict[index], int(body)) + # Uncomment the following line to prove threads overlap. +## sys.stdout.write("%d " % index) + + # Start <request_count> requests from each of + # <client_thread_count> concurrent clients + ts = [] + for c in range(client_thread_count): + data_dict[c] = 0 + t = threading.Thread(target=request, args=(c,)) + ts.append(t) + t.start() + + for t in ts: + t.join() + + hitcount = max(data_dict.values()) + expected = 1 + (client_thread_count * request_count) + + for e in errors: + print(e) + self.assertEqual(hitcount, expected) + + def test_3_Redirect(self): + # Start a new session + self.getPage('/testStr') + self.getPage('/iredir', self.cookies) + self.assertBody("file") + + def test_4_File_deletion(self): + # Start a new session + self.getPage('/testStr') + # Delete the session file manually and retry. + id = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] + path = os.path.join(localDir, "session-" + id) + os.unlink(path) + self.getPage('/testStr', self.cookies) + + def test_5_Error_paths(self): + self.getPage('/unknown/page') + self.assertErrorPage(404, "The path '/unknown/page' was not found.") + + # Note: this path is *not* the same as above. The above + # takes a normal route through the session code; this one + # skips the session code's before_handler and only calls + # before_finalize (save) and on_end (close). So the session + # code has to survive calling save/close without init. + self.getPage('/restricted', self.cookies, method='POST') + self.assertErrorPage(405, response_codes[405]) + + def test_6_regenerate(self): + self.getPage('/testStr') + # grab the cookie ID + id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] + self.getPage('/regen') + self.assertBody('logged in') + id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] + self.assertNotEqual(id1, id2) + + self.getPage('/testStr') + # grab the cookie ID + id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] + self.getPage('/testStr', + headers=[('Cookie', + 'session_id=maliciousid; ' + 'expires=Sat, 27 Oct 2017 04:18:28 GMT; Path=/;')]) + id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] + self.assertNotEqual(id1, id2) + self.assertNotEqual(id2, 'maliciousid') + + def test_7_session_cookies(self): + self.getPage('/setsessiontype/ram') + self.getPage('/clear') + self.getPage('/session_cookie') + # grab the cookie ID + cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) + # Assert there is no 'expires' param + self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) + id1 = cookie_parts['temp'] + self.assertEqual(copykeys(sessions.RamSession.cache), [id1]) + + # Send another request in the same "browser session". + self.getPage('/session_cookie', self.cookies) + cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) + # Assert there is no 'expires' param + self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) + self.assertBody(id1) + self.assertEqual(copykeys(sessions.RamSession.cache), [id1]) + + # Simulate a browser close by just not sending the cookies + self.getPage('/session_cookie') + # grab the cookie ID + cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) + # Assert there is no 'expires' param + self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) + # Assert a new id has been generated... + id2 = cookie_parts['temp'] + self.assertNotEqual(id1, id2) + self.assertEqual(set(sessions.RamSession.cache.keys()), set([id1, id2])) + + # Wait for the session.timeout on both sessions + time.sleep(2.5) + cache = copykeys(sessions.RamSession.cache) + if cache: + if cache == [id2]: + self.fail("The second session did not time out.") + else: + self.fail("Unknown session id in cache: %r", cache) + + +import socket +try: + import memcache + + host, port = '127.0.0.1', 11211 + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + raise + break +except (ImportError, socket.error): + class MemcachedSessionTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test(self): + return self.skip("memcached not reachable ") +else: + class MemcachedSessionTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def test_0_Session(self): + self.getPage('/setsessiontype/memcached') + + self.getPage('/testStr') + self.assertBody('1') + self.getPage('/testGen', self.cookies) + self.assertBody('2') + self.getPage('/testStr', self.cookies) + self.assertBody('3') + self.getPage('/length', self.cookies) + self.assertErrorPage(500) + self.assertInBody("NotImplementedError") + self.getPage('/delkey?key=counter', self.cookies) + self.assertStatus(200) + + # Wait for the session.timeout (1 second) + time.sleep(1.25) + self.getPage('/') + self.assertBody('1') + + # Test session __contains__ + self.getPage('/keyin?key=counter', self.cookies) + self.assertBody("True") + + # Test session delete + self.getPage('/delete', self.cookies) + self.assertBody("done") + + def test_1_Concurrency(self): + client_thread_count = 5 + request_count = 30 + + # Get initial cookie + self.getPage("/") + self.assertBody("1") + cookies = self.cookies + + data_dict = {} + + def request(index): + for i in range(request_count): + self.getPage("/", cookies) + # Uncomment the following line to prove threads overlap. +## sys.stdout.write("%d " % index) + if not self.body.isdigit(): + self.fail(self.body) + data_dict[index] = v = int(self.body) + + # Start <request_count> concurrent requests from + # each of <client_thread_count> clients + ts = [] + for c in range(client_thread_count): + data_dict[c] = 0 + t = threading.Thread(target=request, args=(c,)) + ts.append(t) + t.start() + + for t in ts: + t.join() + + hitcount = max(data_dict.values()) + expected = 1 + (client_thread_count * request_count) + self.assertEqual(hitcount, expected) + + def test_3_Redirect(self): + # Start a new session + self.getPage('/testStr') + self.getPage('/iredir', self.cookies) + self.assertBody("memcached") + + def test_5_Error_paths(self): + self.getPage('/unknown/page') + self.assertErrorPage(404, "The path '/unknown/page' was not found.") + + # Note: this path is *not* the same as above. The above + # takes a normal route through the session code; this one + # skips the session code's before_handler and only calls + # before_finalize (save) and on_end (close). So the session + # code has to survive calling save/close without init. + self.getPage('/restricted', self.cookies, method='POST') + self.assertErrorPage(405, response_codes[405]) + diff --git a/cherrypy/test/test_sessionauthenticate.py b/cherrypy/test/test_sessionauthenticate.py new file mode 100755 index 0000000..ab1fe51 --- /dev/null +++ b/cherrypy/test/test_sessionauthenticate.py @@ -0,0 +1,62 @@ +import cherrypy +from cherrypy.test import helper + + +class SessionAuthenticateTest(helper.CPWebCase): + + def setup_server(): + + def check(username, password): + # Dummy check_username_and_password function + if username != 'test' or password != 'password': + return 'Wrong login/password' + + def augment_params(): + # A simple tool to add some things to request.params + # This is to check to make sure that session_auth can handle request + # params (ticket #780) + cherrypy.request.params["test"] = "test" + + cherrypy.tools.augment_params = cherrypy.Tool('before_handler', + augment_params, None, priority=30) + + class Test: + + _cp_config = {'tools.sessions.on': True, + 'tools.session_auth.on': True, + 'tools.session_auth.check_username_and_password': check, + 'tools.augment_params.on': True, + } + + def index(self, **kwargs): + return "Hi %s, you are logged in" % cherrypy.request.login + index.exposed = True + + cherrypy.tree.mount(Test()) + setup_server = staticmethod(setup_server) + + + def testSessionAuthenticate(self): + # request a page and check for login form + self.getPage('/') + self.assertInBody('<form method="post" action="do_login">') + + # setup credentials + login_body = 'username=test&password=password&from_page=/' + + # attempt a login + self.getPage('/do_login', method='POST', body=login_body) + self.assertStatus((302, 303)) + + # get the page now that we are logged in + self.getPage('/', self.cookies) + self.assertBody('Hi test, you are logged in') + + # do a logout + self.getPage('/do_logout', self.cookies, method='POST') + self.assertStatus((302, 303)) + + # verify we are logged out + self.getPage('/', self.cookies) + self.assertInBody('<form method="post" action="do_login">') + diff --git a/cherrypy/test/test_states.py b/cherrypy/test/test_states.py new file mode 100755 index 0000000..0f97337 --- /dev/null +++ b/cherrypy/test/test_states.py @@ -0,0 +1,436 @@ +from cherrypy._cpcompat import BadStatusLine, ntob +import os +import sys +import threading +import time + +import cherrypy +engine = cherrypy.engine +thisdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + + +class Dependency: + + def __init__(self, bus): + self.bus = bus + self.running = False + self.startcount = 0 + self.gracecount = 0 + self.threads = {} + + def subscribe(self): + self.bus.subscribe('start', self.start) + self.bus.subscribe('stop', self.stop) + self.bus.subscribe('graceful', self.graceful) + self.bus.subscribe('start_thread', self.startthread) + self.bus.subscribe('stop_thread', self.stopthread) + + def start(self): + self.running = True + self.startcount += 1 + + def stop(self): + self.running = False + + def graceful(self): + self.gracecount += 1 + + def startthread(self, thread_id): + self.threads[thread_id] = None + + def stopthread(self, thread_id): + del self.threads[thread_id] + +db_connection = Dependency(engine) + +def setup_server(): + class Root: + def index(self): + return "Hello World" + index.exposed = True + + def ctrlc(self): + raise KeyboardInterrupt() + ctrlc.exposed = True + + def graceful(self): + engine.graceful() + return "app was (gracefully) restarted succesfully" + graceful.exposed = True + + def block_explicit(self): + while True: + if cherrypy.response.timed_out: + cherrypy.response.timed_out = False + return "broken!" + time.sleep(0.01) + block_explicit.exposed = True + + def block_implicit(self): + time.sleep(0.5) + return "response.timeout = %s" % cherrypy.response.timeout + block_implicit.exposed = True + + cherrypy.tree.mount(Root()) + cherrypy.config.update({ + 'environment': 'test_suite', + 'engine.deadlock_poll_freq': 0.1, + }) + + db_connection.subscribe() + + + +# ------------ Enough helpers. Time for real live test cases. ------------ # + + +from cherrypy.test import helper + +class ServerStateTests(helper.CPWebCase): + setup_server = staticmethod(setup_server) + + def setUp(self): + cherrypy.server.socket_timeout = 0.1 + + def test_0_NormalStateFlow(self): + engine.stop() + # Our db_connection should not be running + self.assertEqual(db_connection.running, False) + self.assertEqual(db_connection.startcount, 1) + self.assertEqual(len(db_connection.threads), 0) + + # Test server start + engine.start() + self.assertEqual(engine.state, engine.states.STARTED) + + host = cherrypy.server.socket_host + port = cherrypy.server.socket_port + self.assertRaises(IOError, cherrypy._cpserver.check_port, host, port) + + # The db_connection should be running now + self.assertEqual(db_connection.running, True) + self.assertEqual(db_connection.startcount, 2) + self.assertEqual(len(db_connection.threads), 0) + + self.getPage("/") + self.assertBody("Hello World") + self.assertEqual(len(db_connection.threads), 1) + + # Test engine stop. This will also stop the HTTP server. + engine.stop() + self.assertEqual(engine.state, engine.states.STOPPED) + + # Verify that our custom stop function was called + self.assertEqual(db_connection.running, False) + self.assertEqual(len(db_connection.threads), 0) + + # Block the main thread now and verify that exit() works. + def exittest(): + self.getPage("/") + self.assertBody("Hello World") + engine.exit() + cherrypy.server.start() + engine.start_with_callback(exittest) + engine.block() + self.assertEqual(engine.state, engine.states.EXITING) + + def test_1_Restart(self): + cherrypy.server.start() + engine.start() + + # The db_connection should be running now + self.assertEqual(db_connection.running, True) + grace = db_connection.gracecount + + self.getPage("/") + self.assertBody("Hello World") + self.assertEqual(len(db_connection.threads), 1) + + # Test server restart from this thread + engine.graceful() + self.assertEqual(engine.state, engine.states.STARTED) + self.getPage("/") + self.assertBody("Hello World") + self.assertEqual(db_connection.running, True) + self.assertEqual(db_connection.gracecount, grace + 1) + self.assertEqual(len(db_connection.threads), 1) + + # Test server restart from inside a page handler + self.getPage("/graceful") + self.assertEqual(engine.state, engine.states.STARTED) + self.assertBody("app was (gracefully) restarted succesfully") + self.assertEqual(db_connection.running, True) + self.assertEqual(db_connection.gracecount, grace + 2) + # Since we are requesting synchronously, is only one thread used? + # Note that the "/graceful" request has been flushed. + self.assertEqual(len(db_connection.threads), 0) + + engine.stop() + self.assertEqual(engine.state, engine.states.STOPPED) + self.assertEqual(db_connection.running, False) + self.assertEqual(len(db_connection.threads), 0) + + def test_2_KeyboardInterrupt(self): + # Raise a keyboard interrupt in the HTTP server's main thread. + # We must start the server in this, the main thread + engine.start() + cherrypy.server.start() + + self.persistent = True + try: + # Make the first request and assert there's no "Connection: close". + self.getPage("/") + self.assertStatus('200 OK') + self.assertBody("Hello World") + self.assertNoHeader("Connection") + + cherrypy.server.httpserver.interrupt = KeyboardInterrupt + engine.block() + + self.assertEqual(db_connection.running, False) + self.assertEqual(len(db_connection.threads), 0) + self.assertEqual(engine.state, engine.states.EXITING) + finally: + self.persistent = False + + # Raise a keyboard interrupt in a page handler; on multithreaded + # servers, this should occur in one of the worker threads. + # This should raise a BadStatusLine error, since the worker + # thread will just die without writing a response. + engine.start() + cherrypy.server.start() + + try: + self.getPage("/ctrlc") + except BadStatusLine: + pass + else: + print(self.body) + self.fail("AssertionError: BadStatusLine not raised") + + engine.block() + self.assertEqual(db_connection.running, False) + self.assertEqual(len(db_connection.threads), 0) + + def test_3_Deadlocks(self): + cherrypy.config.update({'response.timeout': 0.2}) + + engine.start() + cherrypy.server.start() + try: + self.assertNotEqual(engine.timeout_monitor.thread, None) + + # Request a "normal" page. + self.assertEqual(engine.timeout_monitor.servings, []) + self.getPage("/") + self.assertBody("Hello World") + # request.close is called async. + while engine.timeout_monitor.servings: + sys.stdout.write(".") + time.sleep(0.01) + + # Request a page that explicitly checks itself for deadlock. + # The deadlock_timeout should be 2 secs. + self.getPage("/block_explicit") + self.assertBody("broken!") + + # Request a page that implicitly breaks deadlock. + # If we deadlock, we want to touch as little code as possible, + # so we won't even call handle_error, just bail ASAP. + self.getPage("/block_implicit") + self.assertStatus(500) + self.assertInBody("raise cherrypy.TimeoutError()") + finally: + engine.exit() + + def test_4_Autoreload(self): + # Start the demo script in a new process + p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) + p.write_conf( + extra='test_case_name: "test_4_Autoreload"') + p.start(imports='cherrypy.test._test_states_demo') + try: + self.getPage("/start") + start = float(self.body) + + # Give the autoreloader time to cache the file time. + time.sleep(2) + + # Touch the file + os.utime(os.path.join(thisdir, "_test_states_demo.py"), None) + + # Give the autoreloader time to re-exec the process + time.sleep(2) + host = cherrypy.server.socket_host + port = cherrypy.server.socket_port + cherrypy._cpserver.wait_for_occupied_port(host, port) + + self.getPage("/start") + self.assert_(float(self.body) > start) + finally: + # Shut down the spawned process + self.getPage("/exit") + p.join() + + def test_5_Start_Error(self): + # If a process errors during start, it should stop the engine + # and exit with a non-zero exit code. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), + wait=True) + p.write_conf( + extra="""starterror: True +test_case_name: "test_5_Start_Error" +""" + ) + p.start(imports='cherrypy.test._test_states_demo') + if p.exit_code == 0: + self.fail("Process failed to return nonzero exit code.") + + +class PluginTests(helper.CPWebCase): + def test_daemonize(self): + if os.name not in ['posix']: + return self.skip("skipped (not on posix) ") + self.HOST = '127.0.0.1' + self.PORT = 8081 + # Spawn the process and wait, when this returns, the original process + # is finished. If it daemonized properly, we should still be able + # to access pages. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), + wait=True, daemonize=True, + socket_host='127.0.0.1', + socket_port=8081) + p.write_conf( + extra='test_case_name: "test_daemonize"') + p.start(imports='cherrypy.test._test_states_demo') + try: + # Just get the pid of the daemonization process. + self.getPage("/pid") + self.assertStatus(200) + page_pid = int(self.body) + self.assertEqual(page_pid, p.get_pid()) + finally: + # Shut down the spawned process + self.getPage("/exit") + p.join() + + # Wait until here to test the exit code because we want to ensure + # that we wait for the daemon to finish running before we fail. + if p.exit_code != 0: + self.fail("Daemonized parent process failed to exit cleanly.") + + +class SignalHandlingTests(helper.CPWebCase): + def test_SIGHUP_tty(self): + # When not daemonized, SIGHUP should shut down the server. + try: + from signal import SIGHUP + except ImportError: + return self.skip("skipped (no SIGHUP) ") + + # Spawn the process. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) + p.write_conf( + extra='test_case_name: "test_SIGHUP_tty"') + p.start(imports='cherrypy.test._test_states_demo') + # Send a SIGHUP + os.kill(p.get_pid(), SIGHUP) + # This might hang if things aren't working right, but meh. + p.join() + + def test_SIGHUP_daemonized(self): + # When daemonized, SIGHUP should restart the server. + try: + from signal import SIGHUP + except ImportError: + return self.skip("skipped (no SIGHUP) ") + + if os.name not in ['posix']: + return self.skip("skipped (not on posix) ") + + # Spawn the process and wait, when this returns, the original process + # is finished. If it daemonized properly, we should still be able + # to access pages. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), + wait=True, daemonize=True) + p.write_conf( + extra='test_case_name: "test_SIGHUP_daemonized"') + p.start(imports='cherrypy.test._test_states_demo') + + pid = p.get_pid() + try: + # Send a SIGHUP + os.kill(pid, SIGHUP) + # Give the server some time to restart + time.sleep(2) + self.getPage("/pid") + self.assertStatus(200) + new_pid = int(self.body) + self.assertNotEqual(new_pid, pid) + finally: + # Shut down the spawned process + self.getPage("/exit") + p.join() + + def test_SIGTERM(self): + # SIGTERM should shut down the server whether daemonized or not. + try: + from signal import SIGTERM + except ImportError: + return self.skip("skipped (no SIGTERM) ") + + try: + from os import kill + except ImportError: + return self.skip("skipped (no os.kill) ") + + # Spawn a normal, undaemonized process. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) + p.write_conf( + extra='test_case_name: "test_SIGTERM"') + p.start(imports='cherrypy.test._test_states_demo') + # Send a SIGTERM + os.kill(p.get_pid(), SIGTERM) + # This might hang if things aren't working right, but meh. + p.join() + + if os.name in ['posix']: + # Spawn a daemonized process and test again. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), + wait=True, daemonize=True) + p.write_conf( + extra='test_case_name: "test_SIGTERM_2"') + p.start(imports='cherrypy.test._test_states_demo') + # Send a SIGTERM + os.kill(p.get_pid(), SIGTERM) + # This might hang if things aren't working right, but meh. + p.join() + + def test_signal_handler_unsubscribe(self): + try: + from signal import SIGTERM + except ImportError: + return self.skip("skipped (no SIGTERM) ") + + try: + from os import kill + except ImportError: + return self.skip("skipped (no os.kill) ") + + # Spawn a normal, undaemonized process. + p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) + p.write_conf( + extra="""unsubsig: True +test_case_name: "test_signal_handler_unsubscribe" +""") + p.start(imports='cherrypy.test._test_states_demo') + # Send a SIGTERM + os.kill(p.get_pid(), SIGTERM) + # This might hang if things aren't working right, but meh. + p.join() + + # Assert the old handler ran. + target_line = open(p.error_log, 'rb').readlines()[-10] + if not ntob("I am an old SIGTERM handler.") in target_line: + self.fail("Old SIGTERM handler did not run.\n%r" % target_line) + diff --git a/cherrypy/test/test_static.py b/cherrypy/test/test_static.py new file mode 100755 index 0000000..871420b --- /dev/null +++ b/cherrypy/test/test_static.py @@ -0,0 +1,300 @@ +from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob +from cherrypy._cpcompat import BytesIO + +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +has_space_filepath = os.path.join(curdir, 'static', 'has space.html') +bigfile_filepath = os.path.join(curdir, "static", "bigfile.log") +BIGFILE_SIZE = 1024 * 1024 +import threading + +import cherrypy +from cherrypy.lib import static +from cherrypy.test import helper + + +class StaticTest(helper.CPWebCase): + + def setup_server(): + if not os.path.exists(has_space_filepath): + open(has_space_filepath, 'wb').write(ntob('Hello, world\r\n')) + if not os.path.exists(bigfile_filepath): + open(bigfile_filepath, 'wb').write(ntob("x" * BIGFILE_SIZE)) + + class Root: + + def bigfile(self): + from cherrypy.lib import static + self.f = static.serve_file(bigfile_filepath) + return self.f + bigfile.exposed = True + bigfile._cp_config = {'response.stream': True} + + def tell(self): + if self.f.input.closed: + return '' + return repr(self.f.input.tell()).rstrip('L') + tell.exposed = True + + def fileobj(self): + f = open(os.path.join(curdir, 'style.css'), 'rb') + return static.serve_fileobj(f, content_type='text/css') + fileobj.exposed = True + + def bytesio(self): + f = BytesIO(ntob('Fee\nfie\nfo\nfum')) + return static.serve_fileobj(f, content_type='text/plain') + bytesio.exposed = True + + class Static: + + def index(self): + return 'You want the Baron? You can have the Baron!' + index.exposed = True + + def dynamic(self): + return "This is a DYNAMIC page" + dynamic.exposed = True + + + root = Root() + root.static = Static() + + rootconf = { + '/static': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.root': curdir, + }, + '/style.css': { + 'tools.staticfile.on': True, + 'tools.staticfile.filename': os.path.join(curdir, 'style.css'), + }, + '/docroot': { + 'tools.staticdir.on': True, + 'tools.staticdir.root': curdir, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.index': 'index.html', + }, + '/error': { + 'tools.staticdir.on': True, + 'request.show_tracebacks': True, + }, + } + rootApp = cherrypy.Application(root) + rootApp.merge(rootconf) + + test_app_conf = { + '/test': { + 'tools.staticdir.index': 'index.html', + 'tools.staticdir.on': True, + 'tools.staticdir.root': curdir, + 'tools.staticdir.dir': 'static', + }, + } + testApp = cherrypy.Application(Static()) + testApp.merge(test_app_conf) + + vhost = cherrypy._cpwsgi.VirtualHost(rootApp, {'virt.net': testApp}) + cherrypy.tree.graft(vhost) + setup_server = staticmethod(setup_server) + + + def teardown_server(): + for f in (has_space_filepath, bigfile_filepath): + if os.path.exists(f): + try: + os.unlink(f) + except: + pass + teardown_server = staticmethod(teardown_server) + + + def testStatic(self): + self.getPage("/static/index.html") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html') + self.assertBody('Hello, world\r\n') + + # Using a staticdir.root value in a subdir... + self.getPage("/docroot/index.html") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html') + self.assertBody('Hello, world\r\n') + + # Check a filename with spaces in it + self.getPage("/static/has%20space.html") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html') + self.assertBody('Hello, world\r\n') + + self.getPage("/style.css") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/css') + # Note: The body should be exactly 'Dummy stylesheet\n', but + # unfortunately some tools such as WinZip sometimes turn \n + # into \r\n on Windows when extracting the CherryPy tarball so + # we just check the content + self.assertMatchesBody('^Dummy stylesheet') + + def test_fallthrough(self): + # Test that NotFound will then try dynamic handlers (see [878]). + self.getPage("/static/dynamic") + self.assertBody("This is a DYNAMIC page") + + # Check a directory via fall-through to dynamic handler. + self.getPage("/static/") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html;charset=utf-8') + self.assertBody('You want the Baron? You can have the Baron!') + + def test_index(self): + # Check a directory via "staticdir.index". + self.getPage("/docroot/") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/html') + self.assertBody('Hello, world\r\n') + # The same page should be returned even if redirected. + self.getPage("/docroot") + self.assertStatus(301) + self.assertHeader('Location', '%s/docroot/' % self.base()) + self.assertMatchesBody("This resource .* <a href='%s/docroot/'>" + "%s/docroot/</a>." % (self.base(), self.base())) + + def test_config_errors(self): + # Check that we get an error if no .file or .dir + self.getPage("/error/thing.html") + self.assertErrorPage(500) + self.assertMatchesBody(ntob("TypeError: staticdir\(\) takes at least 2 " + "(positional )?arguments \(0 given\)")) + + def test_security(self): + # Test up-level security + self.getPage("/static/../../test/style.css") + self.assertStatus((400, 403)) + + def test_modif(self): + # Test modified-since on a reasonably-large file + self.getPage("/static/dirback.jpg") + self.assertStatus("200 OK") + lastmod = "" + for k, v in self.headers: + if k == 'Last-Modified': + lastmod = v + ims = ("If-Modified-Since", lastmod) + self.getPage("/static/dirback.jpg", headers=[ims]) + self.assertStatus(304) + self.assertNoHeader("Content-Type") + self.assertNoHeader("Content-Length") + self.assertNoHeader("Content-Disposition") + self.assertBody("") + + def test_755_vhost(self): + self.getPage("/test/", [('Host', 'virt.net')]) + self.assertStatus(200) + self.getPage("/test", [('Host', 'virt.net')]) + self.assertStatus(301) + self.assertHeader('Location', self.scheme + '://virt.net/test/') + + def test_serve_fileobj(self): + self.getPage("/fileobj") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/css;charset=utf-8') + self.assertMatchesBody('^Dummy stylesheet') + + def test_serve_bytesio(self): + self.getPage("/bytesio") + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/plain;charset=utf-8') + self.assertHeader('Content-Length', 14) + self.assertMatchesBody('Fee\nfie\nfo\nfum') + + def test_file_stream(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Make an initial request + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/bigfile", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + + body = ntob('') + remaining = BIGFILE_SIZE + while remaining > 0: + data = response.fp.read(65536) + if not data: + break + body += data + remaining -= len(data) + + if self.scheme == "https": + newconn = HTTPSConnection + else: + newconn = HTTPConnection + s, h, b = helper.webtest.openURL( + ntob("/tell"), headers=[], host=self.HOST, port=self.PORT, + http_conn=newconn) + if not b: + # The file was closed on the server. + tell_position = BIGFILE_SIZE + else: + tell_position = int(b) + + expected = len(body) + if tell_position >= BIGFILE_SIZE: + # We can't exactly control how much content the server asks for. + # Fudge it by only checking the first half of the reads. + if expected < (BIGFILE_SIZE / 2): + self.fail( + "The file should have advanced to position %r, but has " + "already advanced to the end of the file. It may not be " + "streamed as intended, or at the wrong chunk size (64k)" % + expected) + elif tell_position < expected: + self.fail( + "The file should have advanced to position %r, but has " + "only advanced to position %r. It may not be streamed " + "as intended, or at the wrong chunk size (65536)" % + (expected, tell_position)) + + if body != ntob("x" * BIGFILE_SIZE): + self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % + (BIGFILE_SIZE, body[:50], len(body))) + conn.close() + + def test_file_stream_deadlock(self): + if cherrypy.server.protocol_version != "HTTP/1.1": + return self.skip() + + self.PROTOCOL = "HTTP/1.1" + + # Make an initial request but abort early. + self.persistent = True + conn = self.HTTP_CONN + conn.putrequest("GET", "/bigfile", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + response = conn.response_class(conn.sock, method="GET") + response.begin() + self.assertEqual(response.status, 200) + body = response.fp.read(65536) + if body != ntob("x" * len(body)): + self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % + (65536, body[:50], len(body))) + response.close() + conn.close() + + # Make a second request, which should fetch the whole file. + self.persistent = False + self.getPage("/bigfile") + if self.body != ntob("x" * BIGFILE_SIZE): + self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % + (BIGFILE_SIZE, self.body[:50], len(body))) + diff --git a/cherrypy/test/test_tools.py b/cherrypy/test/test_tools.py new file mode 100755 index 0000000..bc8579f --- /dev/null +++ b/cherrypy/test/test_tools.py @@ -0,0 +1,393 @@ +"""Test the various means of instantiating and invoking tools.""" + +import gzip +import sys +from cherrypy._cpcompat import BytesIO, copyitems, itervalues, IncompleteRead, ntob, ntou, xrange +import time +timeout = 0.2 +import types + +import cherrypy +from cherrypy import tools + + +europoundUnicode = ntou('\x80\xa3') + + +# Client-side code # + +from cherrypy.test import helper + + +class ToolTests(helper.CPWebCase): + def setup_server(): + + # Put check_access in a custom toolbox with its own namespace + myauthtools = cherrypy._cptools.Toolbox("myauth") + + def check_access(default=False): + if not getattr(cherrypy.request, "userid", default): + raise cherrypy.HTTPError(401) + myauthtools.check_access = cherrypy.Tool('before_request_body', check_access) + + def numerify(): + def number_it(body): + for chunk in body: + for k, v in cherrypy.request.numerify_map: + chunk = chunk.replace(k, v) + yield chunk + cherrypy.response.body = number_it(cherrypy.response.body) + + class NumTool(cherrypy.Tool): + def _setup(self): + def makemap(): + m = self._merged_args().get("map", {}) + cherrypy.request.numerify_map = copyitems(m) + cherrypy.request.hooks.attach('on_start_resource', makemap) + + def critical(): + cherrypy.request.error_response = cherrypy.HTTPError(502).set_response + critical.failsafe = True + + cherrypy.request.hooks.attach('on_start_resource', critical) + cherrypy.request.hooks.attach(self._point, self.callable) + + tools.numerify = NumTool('before_finalize', numerify) + + # It's not mandatory to inherit from cherrypy.Tool. + class NadsatTool: + + def __init__(self): + self.ended = {} + self._name = "nadsat" + + def nadsat(self): + def nadsat_it_up(body): + for chunk in body: + chunk = chunk.replace(ntob("good"), ntob("horrorshow")) + chunk = chunk.replace(ntob("piece"), ntob("lomtick")) + yield chunk + cherrypy.response.body = nadsat_it_up(cherrypy.response.body) + nadsat.priority = 0 + + def cleanup(self): + # This runs after the request has been completely written out. + cherrypy.response.body = [ntob("razdrez")] + id = cherrypy.request.params.get("id") + if id: + self.ended[id] = True + cleanup.failsafe = True + + def _setup(self): + cherrypy.request.hooks.attach('before_finalize', self.nadsat) + cherrypy.request.hooks.attach('on_end_request', self.cleanup) + tools.nadsat = NadsatTool() + + def pipe_body(): + cherrypy.request.process_request_body = False + clen = int(cherrypy.request.headers['Content-Length']) + cherrypy.request.body = cherrypy.request.rfile.read(clen) + + # Assert that we can use a callable object instead of a function. + class Rotator(object): + def __call__(self, scale): + r = cherrypy.response + r.collapse_body() + r.body = [chr((ord(x) + scale) % 256) for x in r.body[0]] + cherrypy.tools.rotator = cherrypy.Tool('before_finalize', Rotator()) + + def stream_handler(next_handler, *args, **kwargs): + cherrypy.response.output = o = BytesIO() + try: + response = next_handler(*args, **kwargs) + # Ignore the response and return our accumulated output instead. + return o.getvalue() + finally: + o.close() + cherrypy.tools.streamer = cherrypy._cptools.HandlerWrapperTool(stream_handler) + + class Root: + def index(self): + return "Howdy earth!" + index.exposed = True + + def tarfile(self): + cherrypy.response.output.write(ntob('I am ')) + cherrypy.response.output.write(ntob('a tarfile')) + tarfile.exposed = True + tarfile._cp_config = {'tools.streamer.on': True} + + def euro(self): + hooks = list(cherrypy.request.hooks['before_finalize']) + hooks.sort() + cbnames = [x.callback.__name__ for x in hooks] + assert cbnames == ['gzip'], cbnames + priorities = [x.priority for x in hooks] + assert priorities == [80], priorities + yield ntou("Hello,") + yield ntou("world") + yield europoundUnicode + euro.exposed = True + + # Bare hooks + def pipe(self): + return cherrypy.request.body + pipe.exposed = True + pipe._cp_config = {'hooks.before_request_body': pipe_body} + + # Multiple decorators; include kwargs just for fun. + # Note that rotator must run before gzip. + def decorated_euro(self, *vpath): + yield ntou("Hello,") + yield ntou("world") + yield europoundUnicode + decorated_euro.exposed = True + decorated_euro = tools.gzip(compress_level=6)(decorated_euro) + decorated_euro = tools.rotator(scale=3)(decorated_euro) + + root = Root() + + + class TestType(type): + """Metaclass which automatically exposes all functions in each subclass, + and adds an instance of the subclass as an attribute of root. + """ + def __init__(cls, name, bases, dct): + type.__init__(cls, name, bases, dct) + for value in itervalues(dct): + if isinstance(value, types.FunctionType): + value.exposed = True + setattr(root, name.lower(), cls()) + class Test(object): + __metaclass__ = TestType + + + # METHOD ONE: + # Declare Tools in _cp_config + class Demo(Test): + + _cp_config = {"tools.nadsat.on": True} + + def index(self, id=None): + return "A good piece of cherry pie" + + def ended(self, id): + return repr(tools.nadsat.ended[id]) + + def err(self, id=None): + raise ValueError() + + def errinstream(self, id=None): + yield "nonconfidential" + raise ValueError() + yield "confidential" + + # METHOD TWO: decorator using Tool() + # We support Python 2.3, but the @-deco syntax would look like this: + # @tools.check_access() + def restricted(self): + return "Welcome!" + restricted = myauthtools.check_access()(restricted) + userid = restricted + + def err_in_onstart(self): + return "success!" + + def stream(self, id=None): + for x in xrange(100000000): + yield str(x) + stream._cp_config = {'response.stream': True} + + + conf = { + # METHOD THREE: + # Declare Tools in detached config + '/demo': { + 'tools.numerify.on': True, + 'tools.numerify.map': {ntob("pie"): ntob("3.14159")}, + }, + '/demo/restricted': { + 'request.show_tracebacks': False, + }, + '/demo/userid': { + 'request.show_tracebacks': False, + 'myauth.check_access.default': True, + }, + '/demo/errinstream': { + 'response.stream': True, + }, + '/demo/err_in_onstart': { + # Because this isn't a dict, on_start_resource will error. + 'tools.numerify.map': "pie->3.14159" + }, + # Combined tools + '/euro': { + 'tools.gzip.on': True, + 'tools.encode.on': True, + }, + # Priority specified in config + '/decorated_euro/subpath': { + 'tools.gzip.priority': 10, + }, + # Handler wrappers + '/tarfile': {'tools.streamer.on': True} + } + app = cherrypy.tree.mount(root, config=conf) + app.request_class.namespaces['myauth'] = myauthtools + + if sys.version_info >= (2, 5): + from cherrypy.test import _test_decorators + root.tooldecs = _test_decorators.ToolExamples() + setup_server = staticmethod(setup_server) + + def testHookErrors(self): + self.getPage("/demo/?id=1") + # If body is "razdrez", then on_end_request is being called too early. + self.assertBody("A horrorshow lomtick of cherry 3.14159") + # If this fails, then on_end_request isn't being called at all. + time.sleep(0.1) + self.getPage("/demo/ended/1") + self.assertBody("True") + + valerr = '\n raise ValueError()\nValueError' + self.getPage("/demo/err?id=3") + # If body is "razdrez", then on_end_request is being called too early. + self.assertErrorPage(502, pattern=valerr) + # If this fails, then on_end_request isn't being called at all. + time.sleep(0.1) + self.getPage("/demo/ended/3") + self.assertBody("True") + + # If body is "razdrez", then on_end_request is being called too early. + if (cherrypy.server.protocol_version == "HTTP/1.0" or + getattr(cherrypy.server, "using_apache", False)): + self.getPage("/demo/errinstream?id=5") + # Because this error is raised after the response body has + # started, the status should not change to an error status. + self.assertStatus("200 OK") + self.assertBody("nonconfidential") + else: + # Because this error is raised after the response body has + # started, and because it's chunked output, an error is raised by + # the HTTP client when it encounters incomplete output. + self.assertRaises((ValueError, IncompleteRead), self.getPage, + "/demo/errinstream?id=5") + # If this fails, then on_end_request isn't being called at all. + time.sleep(0.1) + self.getPage("/demo/ended/5") + self.assertBody("True") + + # Test the "__call__" technique (compile-time decorator). + self.getPage("/demo/restricted") + self.assertErrorPage(401) + + # Test compile-time decorator with kwargs from config. + self.getPage("/demo/userid") + self.assertBody("Welcome!") + + def testEndRequestOnDrop(self): + old_timeout = None + try: + httpserver = cherrypy.server.httpserver + old_timeout = httpserver.timeout + except (AttributeError, IndexError): + return self.skip() + + try: + httpserver.timeout = timeout + + # Test that on_end_request is called even if the client drops. + self.persistent = True + try: + conn = self.HTTP_CONN + conn.putrequest("GET", "/demo/stream?id=9", skip_host=True) + conn.putheader("Host", self.HOST) + conn.endheaders() + # Skip the rest of the request and close the conn. This will + # cause the server's active socket to error, which *should* + # result in the request being aborted, and request.close being + # called all the way up the stack (including WSGI middleware), + # eventually calling our on_end_request hook. + finally: + self.persistent = False + time.sleep(timeout * 2) + # Test that the on_end_request hook was called. + self.getPage("/demo/ended/9") + self.assertBody("True") + finally: + if old_timeout is not None: + httpserver.timeout = old_timeout + + def testGuaranteedHooks(self): + # The 'critical' on_start_resource hook is 'failsafe' (guaranteed + # to run even if there are failures in other on_start methods). + # This is NOT true of the other hooks. + # Here, we have set up a failure in NumerifyTool.numerify_map, + # but our 'critical' hook should run and set the error to 502. + self.getPage("/demo/err_in_onstart") + self.assertErrorPage(502) + self.assertInBody("AttributeError: 'str' object has no attribute 'items'") + + def testCombinedTools(self): + expectedResult = (ntou("Hello,world") + europoundUnicode).encode('utf-8') + zbuf = BytesIO() + zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9) + zfile.write(expectedResult) + zfile.close() + + self.getPage("/euro", headers=[("Accept-Encoding", "gzip"), + ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")]) + self.assertInBody(zbuf.getvalue()[:3]) + + zbuf = BytesIO() + zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=6) + zfile.write(expectedResult) + zfile.close() + + self.getPage("/decorated_euro", headers=[("Accept-Encoding", "gzip")]) + self.assertInBody(zbuf.getvalue()[:3]) + + # This returns a different value because gzip's priority was + # lowered in conf, allowing the rotator to run after gzip. + # Of course, we don't want breakage in production apps, + # but it proves the priority was changed. + self.getPage("/decorated_euro/subpath", + headers=[("Accept-Encoding", "gzip")]) + self.assertInBody(''.join([chr((ord(x) + 3) % 256) for x in zbuf.getvalue()])) + + def testBareHooks(self): + content = "bit of a pain in me gulliver" + self.getPage("/pipe", + headers=[("Content-Length", str(len(content))), + ("Content-Type", "text/plain")], + method="POST", body=content) + self.assertBody(content) + + def testHandlerWrapperTool(self): + self.getPage("/tarfile") + self.assertBody("I am a tarfile") + + def testToolWithConfig(self): + if not sys.version_info >= (2, 5): + return self.skip("skipped (Python 2.5+ only)") + + self.getPage('/tooldecs/blah') + self.assertHeader('Content-Type', 'application/data') + + def testWarnToolOn(self): + # get + try: + numon = cherrypy.tools.numerify.on + except AttributeError: + pass + else: + raise AssertionError("Tool.on did not error as it should have.") + + # set + try: + cherrypy.tools.numerify.on = True + except AttributeError: + pass + else: + raise AssertionError("Tool.on did not error as it should have.") + diff --git a/cherrypy/test/test_tutorials.py b/cherrypy/test/test_tutorials.py new file mode 100755 index 0000000..aab2786 --- /dev/null +++ b/cherrypy/test/test_tutorials.py @@ -0,0 +1,201 @@ +import sys + +import cherrypy +from cherrypy.test import helper + + +class TutorialTest(helper.CPWebCase): + + def setup_server(cls): + + conf = cherrypy.config.copy() + + def load_tut_module(name): + """Import or reload tutorial module as needed.""" + cherrypy.config.reset() + cherrypy.config.update(conf) + + target = "cherrypy.tutorial." + name + if target in sys.modules: + module = reload(sys.modules[target]) + else: + module = __import__(target, globals(), locals(), ['']) + # The above import will probably mount a new app at "". + app = cherrypy.tree.apps[""] + + app.root.load_tut_module = load_tut_module + app.root.sessions = sessions + app.root.traceback_setting = traceback_setting + + cls.supervisor.sync_apps() + load_tut_module.exposed = True + + def sessions(): + cherrypy.config.update({"tools.sessions.on": True}) + sessions.exposed = True + + def traceback_setting(): + return repr(cherrypy.request.show_tracebacks) + traceback_setting.exposed = True + + class Dummy: + pass + root = Dummy() + root.load_tut_module = load_tut_module + cherrypy.tree.mount(root) + setup_server = classmethod(setup_server) + + + def test01HelloWorld(self): + self.getPage("/load_tut_module/tut01_helloworld") + self.getPage("/") + self.assertBody('Hello world!') + + def test02ExposeMethods(self): + self.getPage("/load_tut_module/tut02_expose_methods") + self.getPage("/showMessage") + self.assertBody('Hello world!') + + def test03GetAndPost(self): + self.getPage("/load_tut_module/tut03_get_and_post") + + # Try different GET queries + self.getPage("/greetUser?name=Bob") + self.assertBody("Hey Bob, what's up?") + + self.getPage("/greetUser") + self.assertBody('Please enter your name <a href="./">here</a>.') + + self.getPage("/greetUser?name=") + self.assertBody('No, really, enter your name <a href="./">here</a>.') + + # Try the same with POST + self.getPage("/greetUser", method="POST", body="name=Bob") + self.assertBody("Hey Bob, what's up?") + + self.getPage("/greetUser", method="POST", body="name=") + self.assertBody('No, really, enter your name <a href="./">here</a>.') + + def test04ComplexSite(self): + self.getPage("/load_tut_module/tut04_complex_site") + msg = ''' + <p>Here are some extra useful links:</p> + + <ul> + <li><a href="http://del.icio.us">del.icio.us</a></li> + <li><a href="http://www.mornography.de">Hendrik's weblog</a></li> + </ul> + + <p>[<a href="../">Return to links page</a>]</p>''' + self.getPage("/links/extra/") + self.assertBody(msg) + + def test05DerivedObjects(self): + self.getPage("/load_tut_module/tut05_derived_objects") + msg = ''' + <html> + <head> + <title>Another Page</title> + <head> + <body> + <h2>Another Page</h2> + + <p> + And this is the amazing second page! + </p> + + </body> + </html> + ''' + self.getPage("/another/") + self.assertBody(msg) + + def test06DefaultMethod(self): + self.getPage("/load_tut_module/tut06_default_method") + self.getPage('/hendrik') + self.assertBody('Hendrik Mans, CherryPy co-developer & crazy German ' + '(<a href="./">back</a>)') + + def test07Sessions(self): + self.getPage("/load_tut_module/tut07_sessions") + self.getPage("/sessions") + + self.getPage('/') + self.assertBody("\n During your current session, you've viewed this" + "\n page 1 times! Your life is a patio of fun!" + "\n ") + + self.getPage('/', self.cookies) + self.assertBody("\n During your current session, you've viewed this" + "\n page 2 times! Your life is a patio of fun!" + "\n ") + + def test08GeneratorsAndYield(self): + self.getPage("/load_tut_module/tut08_generators_and_yield") + self.getPage('/') + self.assertBody('<html><body><h2>Generators rule!</h2>' + '<h3>List of users:</h3>' + 'Remi<br/>Carlos<br/>Hendrik<br/>Lorenzo Lamas<br/>' + '</body></html>') + + def test09Files(self): + self.getPage("/load_tut_module/tut09_files") + + # Test upload + filesize = 5 + h = [("Content-type", "multipart/form-data; boundary=x"), + ("Content-Length", str(105 + filesize))] + b = '--x\n' + \ + 'Content-Disposition: form-data; name="myFile"; filename="hello.txt"\r\n' + \ + 'Content-Type: text/plain\r\n' + \ + '\r\n' + \ + 'a' * filesize + '\n' + \ + '--x--\n' + self.getPage('/upload', h, "POST", b) + self.assertBody('''<html> + <body> + myFile length: %d<br /> + myFile filename: hello.txt<br /> + myFile mime-type: text/plain + </body> + </html>''' % filesize) + + # Test download + self.getPage('/download') + self.assertStatus("200 OK") + self.assertHeader("Content-Type", "application/x-download") + self.assertHeader("Content-Disposition", + # Make sure the filename is quoted. + 'attachment; filename="pdf_file.pdf"') + self.assertEqual(len(self.body), 85698) + + def test10HTTPErrors(self): + self.getPage("/load_tut_module/tut10_http_errors") + + self.getPage("/") + self.assertInBody("""<a href="toggleTracebacks">""") + self.assertInBody("""<a href="/doesNotExist">""") + self.assertInBody("""<a href="/error?code=403">""") + self.assertInBody("""<a href="/error?code=500">""") + self.assertInBody("""<a href="/messageArg">""") + + self.getPage("/traceback_setting") + setting = self.body + self.getPage("/toggleTracebacks") + self.assertStatus((302, 303)) + self.getPage("/traceback_setting") + self.assertBody(str(not eval(setting))) + + self.getPage("/error?code=500") + self.assertStatus(500) + self.assertInBody("The server encountered an unexpected condition " + "which prevented it from fulfilling the request.") + + self.getPage("/error?code=403") + self.assertStatus(403) + self.assertInBody("<h2>You can't do that!</h2>") + + self.getPage("/messageArg") + self.assertStatus(500) + self.assertInBody("If you construct an HTTPError with a 'message'") + diff --git a/cherrypy/test/test_virtualhost.py b/cherrypy/test/test_virtualhost.py new file mode 100755 index 0000000..d6eed0e --- /dev/null +++ b/cherrypy/test/test_virtualhost.py @@ -0,0 +1,107 @@ +import os +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + +import cherrypy +from cherrypy.test import helper + + +class VirtualHostTest(helper.CPWebCase): + + def setup_server(): + class Root: + def index(self): + return "Hello, world" + index.exposed = True + + def dom4(self): + return "Under construction" + dom4.exposed = True + + def method(self, value): + return "You sent %s" % repr(value) + method.exposed = True + + class VHost: + def __init__(self, sitename): + self.sitename = sitename + + def index(self): + return "Welcome to %s" % self.sitename + index.exposed = True + + def vmethod(self, value): + return "You sent %s" % repr(value) + vmethod.exposed = True + + def url(self): + return cherrypy.url("nextpage") + url.exposed = True + + # Test static as a handler (section must NOT include vhost prefix) + static = cherrypy.tools.staticdir.handler(section='/static', dir=curdir) + + root = Root() + root.mydom2 = VHost("Domain 2") + root.mydom3 = VHost("Domain 3") + hostmap = {'www.mydom2.com': '/mydom2', + 'www.mydom3.com': '/mydom3', + 'www.mydom4.com': '/dom4', + } + cherrypy.tree.mount(root, config={ + '/': {'request.dispatch': cherrypy.dispatch.VirtualHost(**hostmap)}, + # Test static in config (section must include vhost prefix) + '/mydom2/static2': {'tools.staticdir.on': True, + 'tools.staticdir.root': curdir, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.index': 'index.html', + }, + }) + setup_server = staticmethod(setup_server) + + def testVirtualHost(self): + self.getPage("/", [('Host', 'www.mydom1.com')]) + self.assertBody('Hello, world') + self.getPage("/mydom2/", [('Host', 'www.mydom1.com')]) + self.assertBody('Welcome to Domain 2') + + self.getPage("/", [('Host', 'www.mydom2.com')]) + self.assertBody('Welcome to Domain 2') + self.getPage("/", [('Host', 'www.mydom3.com')]) + self.assertBody('Welcome to Domain 3') + self.getPage("/", [('Host', 'www.mydom4.com')]) + self.assertBody('Under construction') + + # Test GET, POST, and positional params + self.getPage("/method?value=root") + self.assertBody("You sent u'root'") + self.getPage("/vmethod?value=dom2+GET", [('Host', 'www.mydom2.com')]) + self.assertBody("You sent u'dom2 GET'") + self.getPage("/vmethod", [('Host', 'www.mydom3.com')], method="POST", + body="value=dom3+POST") + self.assertBody("You sent u'dom3 POST'") + self.getPage("/vmethod/pos", [('Host', 'www.mydom3.com')]) + self.assertBody("You sent 'pos'") + + # Test that cherrypy.url uses the browser url, not the virtual url + self.getPage("/url", [('Host', 'www.mydom2.com')]) + self.assertBody("%s://www.mydom2.com/nextpage" % self.scheme) + + def test_VHost_plus_Static(self): + # Test static as a handler + self.getPage("/static/style.css", [('Host', 'www.mydom2.com')]) + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'text/css;charset=utf-8') + + # Test static in config + self.getPage("/static2/dirback.jpg", [('Host', 'www.mydom2.com')]) + self.assertStatus('200 OK') + self.assertHeader('Content-Type', 'image/jpeg') + + # Test static config with "index" arg + self.getPage("/static2/", [('Host', 'www.mydom2.com')]) + self.assertStatus('200 OK') + self.assertBody('Hello, world\r\n') + # Since tools.trailing_slash is on by default, this should redirect + self.getPage("/static2", [('Host', 'www.mydom2.com')]) + self.assertStatus(301) + diff --git a/cherrypy/test/test_wsgi_ns.py b/cherrypy/test/test_wsgi_ns.py new file mode 100755 index 0000000..d57013c --- /dev/null +++ b/cherrypy/test/test_wsgi_ns.py @@ -0,0 +1,80 @@ +import cherrypy +from cherrypy.test import helper + + +class WSGI_Namespace_Test(helper.CPWebCase): + + def setup_server(): + + class WSGIResponse(object): + + def __init__(self, appresults): + self.appresults = appresults + self.iter = iter(appresults) + + def __iter__(self): + return self + + def next(self): + return self.iter.next() + + def close(self): + if hasattr(self.appresults, "close"): + self.appresults.close() + + + class ChangeCase(object): + + def __init__(self, app, to=None): + self.app = app + self.to = to + + def __call__(self, environ, start_response): + res = self.app(environ, start_response) + class CaseResults(WSGIResponse): + def next(this): + return getattr(this.iter.next(), self.to)() + return CaseResults(res) + + class Replacer(object): + + def __init__(self, app, map={}): + self.app = app + self.map = map + + def __call__(self, environ, start_response): + res = self.app(environ, start_response) + class ReplaceResults(WSGIResponse): + def next(this): + line = this.iter.next() + for k, v in self.map.iteritems(): + line = line.replace(k, v) + return line + return ReplaceResults(res) + + class Root(object): + + def index(self): + return "HellO WoRlD!" + index.exposed = True + + + root_conf = {'wsgi.pipeline': [('replace', Replacer)], + 'wsgi.replace.map': {'L': 'X', 'l': 'r'}, + } + + app = cherrypy.Application(Root()) + app.wsgiapp.pipeline.append(('changecase', ChangeCase)) + app.wsgiapp.config['changecase'] = {'to': 'upper'} + cherrypy.tree.mount(app, config={'/': root_conf}) + setup_server = staticmethod(setup_server) + + + def test_pipeline(self): + if not cherrypy.server.httpserver: + return self.skip() + + self.getPage("/") + # If body is "HEXXO WORXD!", the middleware was applied out of order. + self.assertBody("HERRO WORRD!") + diff --git a/cherrypy/test/test_wsgi_vhost.py b/cherrypy/test/test_wsgi_vhost.py new file mode 100755 index 0000000..abb1a91 --- /dev/null +++ b/cherrypy/test/test_wsgi_vhost.py @@ -0,0 +1,36 @@ +import cherrypy +from cherrypy.test import helper + + +class WSGI_VirtualHost_Test(helper.CPWebCase): + + def setup_server(): + + class ClassOfRoot(object): + + def __init__(self, name): + self.name = name + + def index(self): + return "Welcome to the %s website!" % self.name + index.exposed = True + + + default = cherrypy.Application(None) + + domains = {} + for year in range(1997, 2008): + app = cherrypy.Application(ClassOfRoot('Class of %s' % year)) + domains['www.classof%s.example' % year] = app + + cherrypy.tree.graft(cherrypy._cpwsgi.VirtualHost(default, domains)) + setup_server = staticmethod(setup_server) + + def test_welcome(self): + if not cherrypy.server.using_wsgi: + return self.skip("skipped (not using WSGI)... ") + + for year in range(1997, 2008): + self.getPage("/", headers=[('Host', 'www.classof%s.example' % year)]) + self.assertBody("Welcome to the Class of %s website!" % year) + diff --git a/cherrypy/test/test_wsgiapps.py b/cherrypy/test/test_wsgiapps.py new file mode 100755 index 0000000..fa5420c --- /dev/null +++ b/cherrypy/test/test_wsgiapps.py @@ -0,0 +1,111 @@ +from cherrypy.test import helper + + +class WSGIGraftTests(helper.CPWebCase): + + def setup_server(): + import os + curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + + import cherrypy + + def test_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type', 'text/plain')] + start_response(status, response_headers) + output = ['Hello, world!\n', + 'This is a wsgi app running within CherryPy!\n\n'] + keys = list(environ.keys()) + keys.sort() + for k in keys: + output.append('%s: %s\n' % (k,environ[k])) + return output + + def test_empty_string_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type', 'text/plain')] + start_response(status, response_headers) + return ['Hello', '', ' ', '', 'world'] + + + class WSGIResponse(object): + + def __init__(self, appresults): + self.appresults = appresults + self.iter = iter(appresults) + + def __iter__(self): + return self + + def next(self): + return self.iter.next() + + def close(self): + if hasattr(self.appresults, "close"): + self.appresults.close() + + + class ReversingMiddleware(object): + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + results = app(environ, start_response) + class Reverser(WSGIResponse): + def next(this): + line = list(this.iter.next()) + line.reverse() + return "".join(line) + return Reverser(results) + + class Root: + def index(self): + return "I'm a regular CherryPy page handler!" + index.exposed = True + + + cherrypy.tree.mount(Root()) + + cherrypy.tree.graft(test_app, '/hosted/app1') + cherrypy.tree.graft(test_empty_string_app, '/hosted/app3') + + # Set script_name explicitly to None to signal CP that it should + # be pulled from the WSGI environ each time. + app = cherrypy.Application(Root(), script_name=None) + cherrypy.tree.graft(ReversingMiddleware(app), '/hosted/app2') + setup_server = staticmethod(setup_server) + + wsgi_output = '''Hello, world! +This is a wsgi app running within CherryPy!''' + + def test_01_standard_app(self): + self.getPage("/") + self.assertBody("I'm a regular CherryPy page handler!") + + def test_04_pure_wsgi(self): + import cherrypy + if not cherrypy.server.using_wsgi: + return self.skip("skipped (not using WSGI)... ") + self.getPage("/hosted/app1") + self.assertHeader("Content-Type", "text/plain") + self.assertInBody(self.wsgi_output) + + def test_05_wrapped_cp_app(self): + import cherrypy + if not cherrypy.server.using_wsgi: + return self.skip("skipped (not using WSGI)... ") + self.getPage("/hosted/app2/") + body = list("I'm a regular CherryPy page handler!") + body.reverse() + body = "".join(body) + self.assertInBody(body) + + def test_06_empty_string_app(self): + import cherrypy + if not cherrypy.server.using_wsgi: + return self.skip("skipped (not using WSGI)... ") + self.getPage("/hosted/app3") + self.assertHeader("Content-Type", "text/plain") + self.assertInBody('Hello world') + diff --git a/cherrypy/test/test_xmlrpc.py b/cherrypy/test/test_xmlrpc.py new file mode 100755 index 0000000..c4bf61e --- /dev/null +++ b/cherrypy/test/test_xmlrpc.py @@ -0,0 +1,172 @@ +import sys +from xmlrpclib import DateTime, Fault, ServerProxy, SafeTransport + +class HTTPSTransport(SafeTransport): + """Subclass of SafeTransport to fix sock.recv errors (by using file).""" + + def request(self, host, handler, request_body, verbose=0): + # issue XML-RPC request + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + self.send_content(h, request_body) + + errcode, errmsg, headers = h.getreply() + if errcode != 200: + raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg, + headers) + + self.verbose = verbose + + # Here's where we differ from the superclass. It says: + # try: + # sock = h._conn.sock + # except AttributeError: + # sock = None + # return self._parse_response(h.getfile(), sock) + + return self.parse_response(h.getfile()) + +import cherrypy + + +def setup_server(): + from cherrypy import _cptools + + class Root: + def index(self): + return "I'm a standard index!" + index.exposed = True + + + class XmlRpc(_cptools.XMLRPCController): + + def foo(self): + return "Hello world!" + foo.exposed = True + + def return_single_item_list(self): + return [42] + return_single_item_list.exposed = True + + def return_string(self): + return "here is a string" + return_string.exposed = True + + def return_tuple(self): + return ('here', 'is', 1, 'tuple') + return_tuple.exposed = True + + def return_dict(self): + return dict(a=1, b=2, c=3) + return_dict.exposed = True + + def return_composite(self): + return dict(a=1,z=26), 'hi', ['welcome', 'friend'] + return_composite.exposed = True + + def return_int(self): + return 42 + return_int.exposed = True + + def return_float(self): + return 3.14 + return_float.exposed = True + + def return_datetime(self): + return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) + return_datetime.exposed = True + + def return_boolean(self): + return True + return_boolean.exposed = True + + def test_argument_passing(self, num): + return num * 2 + test_argument_passing.exposed = True + + def test_returning_Fault(self): + return Fault(1, "custom Fault response") + test_returning_Fault.exposed = True + + root = Root() + root.xmlrpc = XmlRpc() + cherrypy.tree.mount(root, config={'/': { + 'request.dispatch': cherrypy.dispatch.XMLRPCDispatcher(), + 'tools.xmlrpc.allow_none': 0, + }}) + + +from cherrypy.test import helper + +class XmlRpcTest(helper.CPWebCase): + setup_server = staticmethod(setup_server) + def testXmlRpc(self): + + scheme = "http" + try: + scheme = self.harness.scheme + except AttributeError: + pass + + if scheme == "https": + url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT) + proxy = ServerProxy(url, transport=HTTPSTransport()) + else: + url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT) + proxy = ServerProxy(url) + + # begin the tests ... + self.getPage("/xmlrpc/foo") + self.assertBody("Hello world!") + + self.assertEqual(proxy.return_single_item_list(), [42]) + self.assertNotEqual(proxy.return_single_item_list(), 'one bazillion') + self.assertEqual(proxy.return_string(), "here is a string") + self.assertEqual(proxy.return_tuple(), list(('here', 'is', 1, 'tuple'))) + self.assertEqual(proxy.return_dict(), {'a': 1, 'c': 3, 'b': 2}) + self.assertEqual(proxy.return_composite(), + [{'a': 1, 'z': 26}, 'hi', ['welcome', 'friend']]) + self.assertEqual(proxy.return_int(), 42) + self.assertEqual(proxy.return_float(), 3.14) + self.assertEqual(proxy.return_datetime(), + DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) + self.assertEqual(proxy.return_boolean(), True) + self.assertEqual(proxy.test_argument_passing(22), 22 * 2) + + # Test an error in the page handler (should raise an xmlrpclib.Fault) + try: + proxy.test_argument_passing({}) + except Exception: + x = sys.exc_info()[1] + self.assertEqual(x.__class__, Fault) + self.assertEqual(x.faultString, ("unsupported operand type(s) " + "for *: 'dict' and 'int'")) + else: + self.fail("Expected xmlrpclib.Fault") + + # http://www.cherrypy.org/ticket/533 + # if a method is not found, an xmlrpclib.Fault should be raised + try: + proxy.non_method() + except Exception: + x = sys.exc_info()[1] + self.assertEqual(x.__class__, Fault) + self.assertEqual(x.faultString, 'method "non_method" is not supported') + else: + self.fail("Expected xmlrpclib.Fault") + + # Test returning a Fault from the page handler. + try: + proxy.test_returning_Fault() + except Exception: + x = sys.exc_info()[1] + self.assertEqual(x.__class__, Fault) + self.assertEqual(x.faultString, ("custom Fault response")) + else: + self.fail("Expected xmlrpclib.Fault") + diff --git a/cherrypy/test/webtest.py b/cherrypy/test/webtest.py new file mode 100755 index 0000000..969eab0 --- /dev/null +++ b/cherrypy/test/webtest.py @@ -0,0 +1,535 @@ +"""Extensions to unittest for web frameworks. + +Use the WebCase.getPage method to request a page from your HTTP server. + +Framework Integration +===================== + +If you have control over your server process, you can handle errors +in the server-side of the HTTP conversation a bit better. You must run +both the client (your WebCase tests) and the server in the same process +(but in separate threads, obviously). + +When an error occurs in the framework, call server_error. It will print +the traceback to stdout, and keep any assertions you have from running +(the assumption is that, if the server errors, the page output will not +be of further significance to your tests). +""" + +import os +import pprint +import re +import socket +import sys +import time +import traceback +import types + +from unittest import * +from unittest import _TextTestResult + +from cherrypy._cpcompat import basestring, HTTPConnection, HTTPSConnection, unicodestr + + + +def interface(host): + """Return an IP address for a client connection given the server host. + + If the server is listening on '0.0.0.0' (INADDR_ANY) + or '::' (IN6ADDR_ANY), this will return the proper localhost.""" + if host == '0.0.0.0': + # INADDR_ANY, which should respond on localhost. + return "127.0.0.1" + if host == '::': + # IN6ADDR_ANY, which should respond on localhost. + return "::1" + return host + + +class TerseTestResult(_TextTestResult): + + def printErrors(self): + # Overridden to avoid unnecessary empty line + if self.errors or self.failures: + if self.dots or self.showAll: + self.stream.writeln() + self.printErrorList('ERROR', self.errors) + self.printErrorList('FAIL', self.failures) + + +class TerseTestRunner(TextTestRunner): + """A test runner class that displays results in textual form.""" + + def _makeResult(self): + return TerseTestResult(self.stream, self.descriptions, self.verbosity) + + def run(self, test): + "Run the given test case or test suite." + # Overridden to remove unnecessary empty lines and separators + result = self._makeResult() + test(result) + result.printErrors() + if not result.wasSuccessful(): + self.stream.write("FAILED (") + failed, errored = list(map(len, (result.failures, result.errors))) + if failed: + self.stream.write("failures=%d" % failed) + if errored: + if failed: self.stream.write(", ") + self.stream.write("errors=%d" % errored) + self.stream.writeln(")") + return result + + +class ReloadingTestLoader(TestLoader): + + def loadTestsFromName(self, name, module=None): + """Return a suite of all tests cases given a string specifier. + + The name may resolve either to a module, a test case class, a + test method within a test case class, or a callable object which + returns a TestCase or TestSuite instance. + + The method optionally resolves the names relative to a given module. + """ + parts = name.split('.') + unused_parts = [] + if module is None: + if not parts: + raise ValueError("incomplete test name: %s" % name) + else: + parts_copy = parts[:] + while parts_copy: + target = ".".join(parts_copy) + if target in sys.modules: + module = reload(sys.modules[target]) + parts = unused_parts + break + else: + try: + module = __import__(target) + parts = unused_parts + break + except ImportError: + unused_parts.insert(0,parts_copy[-1]) + del parts_copy[-1] + if not parts_copy: + raise + parts = parts[1:] + obj = module + for part in parts: + obj = getattr(obj, part) + + if type(obj) == types.ModuleType: + return self.loadTestsFromModule(obj) + elif (isinstance(obj, (type, types.ClassType)) and + issubclass(obj, TestCase)): + return self.loadTestsFromTestCase(obj) + elif type(obj) == types.UnboundMethodType: + return obj.im_class(obj.__name__) + elif hasattr(obj, '__call__'): + test = obj() + if not isinstance(test, TestCase) and \ + not isinstance(test, TestSuite): + raise ValueError("calling %s returned %s, " + "not a test" % (obj,test)) + return test + else: + raise ValueError("do not know how to make test from: %s" % obj) + + +try: + # Jython support + if sys.platform[:4] == 'java': + def getchar(): + # Hopefully this is enough + return sys.stdin.read(1) + else: + # On Windows, msvcrt.getch reads a single char without output. + import msvcrt + def getchar(): + return msvcrt.getch() +except ImportError: + # Unix getchr + import tty, termios + def getchar(): + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + +class WebCase(TestCase): + HOST = "127.0.0.1" + PORT = 8000 + HTTP_CONN = HTTPConnection + PROTOCOL = "HTTP/1.1" + + scheme = "http" + url = None + + status = None + headers = None + body = None + + encoding = 'utf-8' + + time = None + + def get_conn(self, auto_open=False): + """Return a connection to our HTTP server.""" + if self.scheme == "https": + cls = HTTPSConnection + else: + cls = HTTPConnection + conn = cls(self.interface(), self.PORT) + # Automatically re-connect? + conn.auto_open = auto_open + conn.connect() + return conn + + def set_persistent(self, on=True, auto_open=False): + """Make our HTTP_CONN persistent (or not). + + If the 'on' argument is True (the default), then self.HTTP_CONN + will be set to an instance of HTTPConnection (or HTTPS + if self.scheme is "https"). This will then persist across requests. + + We only allow for a single open connection, so if you call this + and we currently have an open connection, it will be closed. + """ + try: + self.HTTP_CONN.close() + except (TypeError, AttributeError): + pass + + if on: + self.HTTP_CONN = self.get_conn(auto_open=auto_open) + else: + if self.scheme == "https": + self.HTTP_CONN = HTTPSConnection + else: + self.HTTP_CONN = HTTPConnection + + def _get_persistent(self): + return hasattr(self.HTTP_CONN, "__class__") + def _set_persistent(self, on): + self.set_persistent(on) + persistent = property(_get_persistent, _set_persistent) + + def interface(self): + """Return an IP address for a client connection. + + If the server is listening on '0.0.0.0' (INADDR_ANY) + or '::' (IN6ADDR_ANY), this will return the proper localhost.""" + return interface(self.HOST) + + def getPage(self, url, headers=None, method="GET", body=None, protocol=None): + """Open the url with debugging support. Return status, headers, body.""" + ServerError.on = False + + if isinstance(url, unicodestr): + url = url.encode('utf-8') + if isinstance(body, unicodestr): + body = body.encode('utf-8') + + self.url = url + self.time = None + start = time.time() + result = openURL(url, headers, method, body, self.HOST, self.PORT, + self.HTTP_CONN, protocol or self.PROTOCOL) + self.time = time.time() - start + self.status, self.headers, self.body = result + + # Build a list of request cookies from the previous response cookies. + self.cookies = [('Cookie', v) for k, v in self.headers + if k.lower() == 'set-cookie'] + + if ServerError.on: + raise ServerError() + return result + + interactive = True + console_height = 30 + + def _handlewebError(self, msg): + print("") + print(" ERROR: %s" % msg) + + if not self.interactive: + raise self.failureException(msg) + + p = " Show: [B]ody [H]eaders [S]tatus [U]RL; [I]gnore, [R]aise, or sys.e[X]it >> " + sys.stdout.write(p) + sys.stdout.flush() + while True: + i = getchar().upper() + if i not in "BHSUIRX": + continue + print(i.upper()) # Also prints new line + if i == "B": + for x, line in enumerate(self.body.splitlines()): + if (x + 1) % self.console_height == 0: + # The \r and comma should make the next line overwrite + sys.stdout.write("<-- More -->\r") + m = getchar().lower() + # Erase our "More" prompt + sys.stdout.write(" \r") + if m == "q": + break + print(line) + elif i == "H": + pprint.pprint(self.headers) + elif i == "S": + print(self.status) + elif i == "U": + print(self.url) + elif i == "I": + # return without raising the normal exception + return + elif i == "R": + raise self.failureException(msg) + elif i == "X": + self.exit() + sys.stdout.write(p) + sys.stdout.flush() + + def exit(self): + sys.exit() + + def assertStatus(self, status, msg=None): + """Fail if self.status != status.""" + if isinstance(status, basestring): + if not self.status == status: + if msg is None: + msg = 'Status (%r) != %r' % (self.status, status) + self._handlewebError(msg) + elif isinstance(status, int): + code = int(self.status[:3]) + if code != status: + if msg is None: + msg = 'Status (%r) != %r' % (self.status, status) + self._handlewebError(msg) + else: + # status is a tuple or list. + match = False + for s in status: + if isinstance(s, basestring): + if self.status == s: + match = True + break + elif int(self.status[:3]) == s: + match = True + break + if not match: + if msg is None: + msg = 'Status (%r) not in %r' % (self.status, status) + self._handlewebError(msg) + + def assertHeader(self, key, value=None, msg=None): + """Fail if (key, [value]) not in self.headers.""" + lowkey = key.lower() + for k, v in self.headers: + if k.lower() == lowkey: + if value is None or str(value) == v: + return v + + if msg is None: + if value is None: + msg = '%r not in headers' % key + else: + msg = '%r:%r not in headers' % (key, value) + self._handlewebError(msg) + + def assertHeaderItemValue(self, key, value, msg=None): + """Fail if the header does not contain the specified value""" + actual_value = self.assertHeader(key, msg=msg) + header_values = map(str.strip, actual_value.split(',')) + if value in header_values: + return value + + if msg is None: + msg = "%r not in %r" % (value, header_values) + self._handlewebError(msg) + + def assertNoHeader(self, key, msg=None): + """Fail if key in self.headers.""" + lowkey = key.lower() + matches = [k for k, v in self.headers if k.lower() == lowkey] + if matches: + if msg is None: + msg = '%r in headers' % key + self._handlewebError(msg) + + def assertBody(self, value, msg=None): + """Fail if value != self.body.""" + if value != self.body: + if msg is None: + msg = 'expected body:\n%r\n\nactual body:\n%r' % (value, self.body) + self._handlewebError(msg) + + def assertInBody(self, value, msg=None): + """Fail if value not in self.body.""" + if value not in self.body: + if msg is None: + msg = '%r not in body: %s' % (value, self.body) + self._handlewebError(msg) + + def assertNotInBody(self, value, msg=None): + """Fail if value in self.body.""" + if value in self.body: + if msg is None: + msg = '%r found in body' % value + self._handlewebError(msg) + + def assertMatchesBody(self, pattern, msg=None, flags=0): + """Fail if value (a regex pattern) is not in self.body.""" + if re.search(pattern, self.body, flags) is None: + if msg is None: + msg = 'No match for %r in body' % pattern + self._handlewebError(msg) + + +methods_with_bodies = ("POST", "PUT") + +def cleanHeaders(headers, method, body, host, port): + """Return request headers, with required headers added (if missing).""" + if headers is None: + headers = [] + + # Add the required Host request header if not present. + # [This specifies the host:port of the server, not the client.] + found = False + for k, v in headers: + if k.lower() == 'host': + found = True + break + if not found: + if port == 80: + headers.append(("Host", host)) + else: + headers.append(("Host", "%s:%s" % (host, port))) + + if method in methods_with_bodies: + # Stick in default type and length headers if not present + found = False + for k, v in headers: + if k.lower() == 'content-type': + found = True + break + if not found: + headers.append(("Content-Type", "application/x-www-form-urlencoded")) + headers.append(("Content-Length", str(len(body or "")))) + + return headers + + +def shb(response): + """Return status, headers, body the way we like from a response.""" + h = [] + key, value = None, None + for line in response.msg.headers: + if line: + if line[0] in " \t": + value += line.strip() + else: + if key and value: + h.append((key, value)) + key, value = line.split(":", 1) + key = key.strip() + value = value.strip() + if key and value: + h.append((key, value)) + + return "%s %s" % (response.status, response.reason), h, response.read() + + +def openURL(url, headers=None, method="GET", body=None, + host="127.0.0.1", port=8000, http_conn=HTTPConnection, + protocol="HTTP/1.1"): + """Open the given HTTP resource and return status, headers, and body.""" + + headers = cleanHeaders(headers, method, body, host, port) + + # Trying 10 times is simply in case of socket errors. + # Normal case--it should run once. + for trial in range(10): + try: + # Allow http_conn to be a class or an instance + if hasattr(http_conn, "host"): + conn = http_conn + else: + conn = http_conn(interface(host), port) + + conn._http_vsn_str = protocol + conn._http_vsn = int("".join([x for x in protocol if x.isdigit()])) + + # skip_accept_encoding argument added in python version 2.4 + if sys.version_info < (2, 4): + def putheader(self, header, value): + if header == 'Accept-Encoding' and value == 'identity': + return + self.__class__.putheader(self, header, value) + import new + conn.putheader = new.instancemethod(putheader, conn, conn.__class__) + conn.putrequest(method.upper(), url, skip_host=True) + else: + conn.putrequest(method.upper(), url, skip_host=True, + skip_accept_encoding=True) + + for key, value in headers: + conn.putheader(key, value) + conn.endheaders() + + if body is not None: + conn.send(body) + + # Handle response + response = conn.getresponse() + + s, h, b = shb(response) + + if not hasattr(http_conn, "host"): + # We made our own conn instance. Close it. + conn.close() + + return s, h, b + except socket.error: + time.sleep(0.5) + raise + + +# Add any exceptions which your web framework handles +# normally (that you don't want server_error to trap). +ignored_exceptions = [] + +# You'll want set this to True when you can't guarantee +# that each response will immediately follow each request; +# for example, when handling requests via multiple threads. +ignore_all = False + +class ServerError(Exception): + on = False + + +def server_error(exc=None): + """Server debug hook. Return True if exception handled, False if ignored. + + You probably want to wrap this, so you can still handle an error using + your framework when it's ignored. + """ + if exc is None: + exc = sys.exc_info() + + if ignore_all or exc[0] in ignored_exceptions: + return False + else: + ServerError.on = True + print("") + print("".join(traceback.format_exception(*exc))) + return True + diff --git a/cherrypy/tutorial/README.txt b/cherrypy/tutorial/README.txt new file mode 100644 index 0000000..2b877e1 --- /dev/null +++ b/cherrypy/tutorial/README.txt @@ -0,0 +1,16 @@ +CherryPy Tutorials
+------------------------------------------------------------------------
+
+This is a series of tutorials explaining how to develop dynamic web
+applications using CherryPy. A couple of notes:
+
+ - Each of these tutorials builds on the ones before it. If you're
+ new to CherryPy, we recommend you start with 01_helloworld.py and
+ work your way upwards. :)
+
+ - In most of these tutorials, you will notice that all output is done
+ by returning normal Python strings, often using simple Python
+ variable substitution. In most real-world applications, you will
+ probably want to use a separate template package (like Cheetah,
+ CherryTemplate or XML/XSL).
+
diff --git a/cherrypy/tutorial/__init__.py b/cherrypy/tutorial/__init__.py new file mode 100755 index 0000000..c4e2c55 --- /dev/null +++ b/cherrypy/tutorial/__init__.py @@ -0,0 +1,3 @@ + +# This is used in test_config to test unrepr of "from A import B" +thing2 = object()
\ No newline at end of file diff --git a/cherrypy/tutorial/bonus-sqlobject.py b/cherrypy/tutorial/bonus-sqlobject.py new file mode 100755 index 0000000..c43feb4 --- /dev/null +++ b/cherrypy/tutorial/bonus-sqlobject.py @@ -0,0 +1,168 @@ +''' +Bonus Tutorial: Using SQLObject + +This is a silly little contacts manager application intended to +demonstrate how to use SQLObject from within a CherryPy2 project. It +also shows how to use inline Cheetah templates. + +SQLObject is an Object/Relational Mapper that allows you to access +data stored in an RDBMS in a pythonic fashion. You create data objects +as Python classes and let SQLObject take care of all the nasty details. + +This code depends on the latest development version (0.6+) of SQLObject. +You can get it from the SQLObject Subversion server. You can find all +necessary information at <http://www.sqlobject.org>. This code will NOT +work with the 0.5.x version advertised on their website! + +This code also depends on a recent version of Cheetah. You can find +Cheetah at <http://www.cheetahtemplate.org>. + +After starting this application for the first time, you will need to +access the /reset URI in order to create the database table and some +sample data. Accessing /reset again will drop and re-create the table, +so you may want to be careful. :-) + +This application isn't supposed to be fool-proof, it's not even supposed +to be very GOOD. Play around with it some, browse the source code, smile. + +:) + +-- Hendrik Mans <hendrik@mans.de> +''' + +import cherrypy +from Cheetah.Template import Template +from sqlobject import * + +# configure your database connection here +__connection__ = 'mysql://root:@localhost/test' + +# this is our (only) data class. +class Contact(SQLObject): + lastName = StringCol(length = 50, notNone = True) + firstName = StringCol(length = 50, notNone = True) + phone = StringCol(length = 30, notNone = True, default = '') + email = StringCol(length = 30, notNone = True, default = '') + url = StringCol(length = 100, notNone = True, default = '') + + +class ContactManager: + def index(self): + # Let's display a list of all stored contacts. + contacts = Contact.select() + + template = Template(''' + <h2>All Contacts</h2> + + #for $contact in $contacts + <a href="mailto:$contact.email">$contact.lastName, $contact.firstName</a> + [<a href="./edit?id=$contact.id">Edit</a>] + [<a href="./delete?id=$contact.id">Delete</a>] + <br/> + #end for + + <p>[<a href="./edit">Add new contact</a>]</p> + ''', [locals(), globals()]) + + return template.respond() + + index.exposed = True + + + def edit(self, id = 0): + # we really want id as an integer. Since GET/POST parameters + # are always passed as strings, let's convert it. + id = int(id) + + if id > 0: + # if an id is specified, we're editing an existing contact. + contact = Contact.get(id) + title = "Edit Contact" + else: + # if no id is specified, we're entering a new contact. + contact = None + title = "New Contact" + + + # In the following template code, please note that we use + # Cheetah's $getVar() construct for the form values. We have + # to do this because contact may be set to None (see above). + template = Template(''' + <h2>$title</h2> + + <form action="./store" method="POST"> + <input type="hidden" name="id" value="$id" /> + Last Name: <input name="lastName" value="$getVar('contact.lastName', '')" /><br/> + First Name: <input name="firstName" value="$getVar('contact.firstName', '')" /><br/> + Phone: <input name="phone" value="$getVar('contact.phone', '')" /><br/> + Email: <input name="email" value="$getVar('contact.email', '')" /><br/> + URL: <input name="url" value="$getVar('contact.url', '')" /><br/> + <input type="submit" value="Store" /> + </form> + ''', [locals(), globals()]) + + return template.respond() + + edit.exposed = True + + + def delete(self, id): + # Delete the specified contact + contact = Contact.get(int(id)) + contact.destroySelf() + return 'Deleted. <a href="./">Return to Index</a>' + + delete.exposed = True + + + def store(self, lastName, firstName, phone, email, url, id = None): + if id and int(id) > 0: + # If an id was specified, update an existing contact. + contact = Contact.get(int(id)) + + # We could set one field after another, but that would + # cause multiple UPDATE clauses. So we'll just do it all + # in a single pass through the set() method. + contact.set( + lastName = lastName, + firstName = firstName, + phone = phone, + email = email, + url = url) + else: + # Otherwise, add a new contact. + contact = Contact( + lastName = lastName, + firstName = firstName, + phone = phone, + email = email, + url = url) + + return 'Stored. <a href="./">Return to Index</a>' + + store.exposed = True + + + def reset(self): + # Drop existing table + Contact.dropTable(True) + + # Create new table + Contact.createTable() + + # Create some sample data + Contact( + firstName = 'Hendrik', + lastName = 'Mans', + email = 'hendrik@mans.de', + phone = '++49 89 12345678', + url = 'http://www.mornography.de') + + return "reset completed!" + + reset.exposed = True + + +print("If you're running this application for the first time, please go to http://localhost:8080/reset once in order to create the database!") + +cherrypy.quickstart(ContactManager()) diff --git a/cherrypy/tutorial/custom_error.html b/cherrypy/tutorial/custom_error.html new file mode 100644 index 0000000..d0f30c8 --- /dev/null +++ b/cherrypy/tutorial/custom_error.html @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html> +<head> + <title>403 Unauthorized</title> +</head> + <body> + <h2>You can't do that!</h2> + <p>%(message)s</p> + <p>This is a custom error page that is read from a file.<p> + <pre>%(traceback)s</pre> + </body> +</html> diff --git a/cherrypy/tutorial/pdf_file.pdf b/cherrypy/tutorial/pdf_file.pdf Binary files differnew file mode 100644 index 0000000..38b4f15 --- /dev/null +++ b/cherrypy/tutorial/pdf_file.pdf diff --git a/cherrypy/tutorial/tut01_helloworld.py b/cherrypy/tutorial/tut01_helloworld.py new file mode 100755 index 0000000..ef94760 --- /dev/null +++ b/cherrypy/tutorial/tut01_helloworld.py @@ -0,0 +1,35 @@ +""" +Tutorial - Hello World + +The most basic (working) CherryPy application possible. +""" + +# Import CherryPy global namespace +import cherrypy + +class HelloWorld: + """ Sample request handler class. """ + + def index(self): + # CherryPy will call this method for the root URI ("/") and send + # its return value to the client. Because this is tutorial + # lesson number 01, we'll just send something really simple. + # How about... + return "Hello world!" + + # Expose the index method through the web. CherryPy will never + # publish methods that don't have the exposed attribute set to True. + index.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(HelloWorld(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(HelloWorld(), config=tutconf) diff --git a/cherrypy/tutorial/tut02_expose_methods.py b/cherrypy/tutorial/tut02_expose_methods.py new file mode 100755 index 0000000..600fca3 --- /dev/null +++ b/cherrypy/tutorial/tut02_expose_methods.py @@ -0,0 +1,32 @@ +""" +Tutorial - Multiple methods + +This tutorial shows you how to link to other methods of your request +handler. +""" + +import cherrypy + +class HelloWorld: + + def index(self): + # Let's link to another method here. + return 'We have an <a href="showMessage">important message</a> for you!' + index.exposed = True + + def showMessage(self): + # Here's the important message! + return "Hello world!" + showMessage.exposed = True + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(HelloWorld(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(HelloWorld(), config=tutconf) diff --git a/cherrypy/tutorial/tut03_get_and_post.py b/cherrypy/tutorial/tut03_get_and_post.py new file mode 100755 index 0000000..283477d --- /dev/null +++ b/cherrypy/tutorial/tut03_get_and_post.py @@ -0,0 +1,53 @@ +""" +Tutorial - Passing variables + +This tutorial shows you how to pass GET/POST variables to methods. +""" + +import cherrypy + + +class WelcomePage: + + def index(self): + # Ask for the user's name. + return ''' + <form action="greetUser" method="GET"> + What is your name? + <input type="text" name="name" /> + <input type="submit" /> + </form>''' + index.exposed = True + + def greetUser(self, name = None): + # CherryPy passes all GET and POST variables as method parameters. + # It doesn't make a difference where the variables come from, how + # large their contents are, and so on. + # + # You can define default parameter values as usual. In this + # example, the "name" parameter defaults to None so we can check + # if a name was actually specified. + + if name: + # Greet the user! + return "Hey %s, what's up?" % name + else: + if name is None: + # No name was specified + return 'Please enter your name <a href="./">here</a>.' + else: + return 'No, really, enter your name <a href="./">here</a>.' + greetUser.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(WelcomePage(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(WelcomePage(), config=tutconf) diff --git a/cherrypy/tutorial/tut04_complex_site.py b/cherrypy/tutorial/tut04_complex_site.py new file mode 100755 index 0000000..b4d820e --- /dev/null +++ b/cherrypy/tutorial/tut04_complex_site.py @@ -0,0 +1,98 @@ +""" +Tutorial - Multiple objects + +This tutorial shows you how to create a site structure through multiple +possibly nested request handler objects. +""" + +import cherrypy + + +class HomePage: + def index(self): + return ''' + <p>Hi, this is the home page! Check out the other + fun stuff on this site:</p> + + <ul> + <li><a href="/joke/">A silly joke</a></li> + <li><a href="/links/">Useful links</a></li> + </ul>''' + index.exposed = True + + +class JokePage: + def index(self): + return ''' + <p>"In Python, how do you create a string of random + characters?" -- "Read a Perl file!"</p> + <p>[<a href="../">Return</a>]</p>''' + index.exposed = True + + +class LinksPage: + def __init__(self): + # Request handler objects can create their own nested request + # handler objects. Simply create them inside their __init__ + # methods! + self.extra = ExtraLinksPage() + + def index(self): + # Note the way we link to the extra links page (and back). + # As you can see, this object doesn't really care about its + # absolute position in the site tree, since we use relative + # links exclusively. + return ''' + <p>Here are some useful links:</p> + + <ul> + <li><a href="http://www.cherrypy.org">The CherryPy Homepage</a></li> + <li><a href="http://www.python.org">The Python Homepage</a></li> + </ul> + + <p>You can check out some extra useful + links <a href="./extra/">here</a>.</p> + + <p>[<a href="../">Return</a>]</p> + ''' + index.exposed = True + + +class ExtraLinksPage: + def index(self): + # Note the relative link back to the Links page! + return ''' + <p>Here are some extra useful links:</p> + + <ul> + <li><a href="http://del.icio.us">del.icio.us</a></li> + <li><a href="http://www.mornography.de">Hendrik's weblog</a></li> + </ul> + + <p>[<a href="../">Return to links page</a>]</p>''' + index.exposed = True + + +# Of course we can also mount request handler objects right here! +root = HomePage() +root.joke = JokePage() +root.links = LinksPage() + +# Remember, we don't need to mount ExtraLinksPage here, because +# LinksPage does that itself on initialization. In fact, there is +# no reason why you shouldn't let your root object take care of +# creating all contained request handler objects. + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(root, config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(root, config=tutconf) + diff --git a/cherrypy/tutorial/tut05_derived_objects.py b/cherrypy/tutorial/tut05_derived_objects.py new file mode 100755 index 0000000..3d4ec9b --- /dev/null +++ b/cherrypy/tutorial/tut05_derived_objects.py @@ -0,0 +1,83 @@ +""" +Tutorial - Object inheritance + +You are free to derive your request handler classes from any base +class you wish. In most real-world applications, you will probably +want to create a central base class used for all your pages, which takes +care of things like printing a common page header and footer. +""" + +import cherrypy + + +class Page: + # Store the page title in a class attribute + title = 'Untitled Page' + + def header(self): + return ''' + <html> + <head> + <title>%s</title> + <head> + <body> + <h2>%s</h2> + ''' % (self.title, self.title) + + def footer(self): + return ''' + </body> + </html> + ''' + + # Note that header and footer don't get their exposed attributes + # set to True. This isn't necessary since the user isn't supposed + # to call header or footer directly; instead, we'll call them from + # within the actually exposed handler methods defined in this + # class' subclasses. + + +class HomePage(Page): + # Different title for this page + title = 'Tutorial 5' + + def __init__(self): + # create a subpage + self.another = AnotherPage() + + def index(self): + # Note that we call the header and footer methods inherited + # from the Page class! + return self.header() + ''' + <p> + Isn't this exciting? There's + <a href="./another/">another page</a>, too! + </p> + ''' + self.footer() + index.exposed = True + + +class AnotherPage(Page): + title = 'Another Page' + + def index(self): + return self.header() + ''' + <p> + And this is the amazing second page! + </p> + ''' + self.footer() + index.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(HomePage(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(HomePage(), config=tutconf) + diff --git a/cherrypy/tutorial/tut06_default_method.py b/cherrypy/tutorial/tut06_default_method.py new file mode 100755 index 0000000..fe24f38 --- /dev/null +++ b/cherrypy/tutorial/tut06_default_method.py @@ -0,0 +1,64 @@ +""" +Tutorial - The default method + +Request handler objects can implement a method called "default" that +is called when no other suitable method/object could be found. +Essentially, if CherryPy2 can't find a matching request handler object +for the given request URI, it will use the default method of the object +located deepest on the URI path. + +Using this mechanism you can easily simulate virtual URI structures +by parsing the extra URI string, which you can access through +cherrypy.request.virtualPath. + +The application in this tutorial simulates an URI structure looking +like /users/<username>. Since the <username> bit will not be found (as +there are no matching methods), it is handled by the default method. +""" + +import cherrypy + + +class UsersPage: + + def index(self): + # Since this is just a stupid little example, we'll simply + # display a list of links to random, made-up users. In a real + # application, this could be generated from a database result set. + return ''' + <a href="./remi">Remi Delon</a><br/> + <a href="./hendrik">Hendrik Mans</a><br/> + <a href="./lorenzo">Lorenzo Lamas</a><br/> + ''' + index.exposed = True + + def default(self, user): + # Here we react depending on the virtualPath -- the part of the + # path that could not be mapped to an object method. In a real + # application, we would probably do some database lookups here + # instead of the silly if/elif/else construct. + if user == 'remi': + out = "Remi Delon, CherryPy lead developer" + elif user == 'hendrik': + out = "Hendrik Mans, CherryPy co-developer & crazy German" + elif user == 'lorenzo': + out = "Lorenzo Lamas, famous actor and singer!" + else: + out = "Unknown user. :-(" + + return '%s (<a href="./">back</a>)' % out + default.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(UsersPage(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(UsersPage(), config=tutconf) + diff --git a/cherrypy/tutorial/tut07_sessions.py b/cherrypy/tutorial/tut07_sessions.py new file mode 100755 index 0000000..4b1386b --- /dev/null +++ b/cherrypy/tutorial/tut07_sessions.py @@ -0,0 +1,44 @@ +""" +Tutorial - Sessions + +Storing session data in CherryPy applications is very easy: cherrypy +provides a dictionary called "session" that represents the session +data for the current user. If you use RAM based sessions, you can store +any kind of object into that dictionary; otherwise, you are limited to +objects that can be pickled. +""" + +import cherrypy + + +class HitCounter: + + _cp_config = {'tools.sessions.on': True} + + def index(self): + # Increase the silly hit counter + count = cherrypy.session.get('count', 0) + 1 + + # Store the new value in the session dictionary + cherrypy.session['count'] = count + + # And display a silly hit count message! + return ''' + During your current session, you've viewed this + page %s times! Your life is a patio of fun! + ''' % count + index.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(HitCounter(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(HitCounter(), config=tutconf) + diff --git a/cherrypy/tutorial/tut08_generators_and_yield.py b/cherrypy/tutorial/tut08_generators_and_yield.py new file mode 100755 index 0000000..a6fbdc2 --- /dev/null +++ b/cherrypy/tutorial/tut08_generators_and_yield.py @@ -0,0 +1,47 @@ +""" +Bonus Tutorial: Using generators to return result bodies + +Instead of returning a complete result string, you can use the yield +statement to return one result part after another. This may be convenient +in situations where using a template package like CherryPy or Cheetah +would be overkill, and messy string concatenation too uncool. ;-) +""" + +import cherrypy + + +class GeneratorDemo: + + def header(self): + return "<html><body><h2>Generators rule!</h2>" + + def footer(self): + return "</body></html>" + + def index(self): + # Let's make up a list of users for presentation purposes + users = ['Remi', 'Carlos', 'Hendrik', 'Lorenzo Lamas'] + + # Every yield line adds one part to the total result body. + yield self.header() + yield "<h3>List of users:</h3>" + + for user in users: + yield "%s<br/>" % user + + yield self.footer() + index.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(GeneratorDemo(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(GeneratorDemo(), config=tutconf) + diff --git a/cherrypy/tutorial/tut09_files.py b/cherrypy/tutorial/tut09_files.py new file mode 100755 index 0000000..4c8e581 --- /dev/null +++ b/cherrypy/tutorial/tut09_files.py @@ -0,0 +1,107 @@ +""" + +Tutorial: File upload and download + +Uploads +------- + +When a client uploads a file to a CherryPy application, it's placed +on disk immediately. CherryPy will pass it to your exposed method +as an argument (see "myFile" below); that arg will have a "file" +attribute, which is a handle to the temporary uploaded file. +If you wish to permanently save the file, you need to read() +from myFile.file and write() somewhere else. + +Note the use of 'enctype="multipart/form-data"' and 'input type="file"' +in the HTML which the client uses to upload the file. + + +Downloads +--------- + +If you wish to send a file to the client, you have two options: +First, you can simply return a file-like object from your page handler. +CherryPy will read the file and serve it as the content (HTTP body) +of the response. However, that doesn't tell the client that +the response is a file to be saved, rather than displayed. +Use cherrypy.lib.static.serve_file for that; it takes four +arguments: + +serve_file(path, content_type=None, disposition=None, name=None) + +Set "name" to the filename that you expect clients to use when they save +your file. Note that the "name" argument is ignored if you don't also +provide a "disposition" (usually "attachement"). You can manually set +"content_type", but be aware that if you also use the encoding tool, it +may choke if the file extension is not recognized as belonging to a known +Content-Type. Setting the content_type to "application/x-download" works +in most cases, and should prompt the user with an Open/Save dialog in +popular browsers. + +""" + +import os +localDir = os.path.dirname(__file__) +absDir = os.path.join(os.getcwd(), localDir) + +import cherrypy +from cherrypy.lib import static + + +class FileDemo(object): + + def index(self): + return """ + <html><body> + <h2>Upload a file</h2> + <form action="upload" method="post" enctype="multipart/form-data"> + filename: <input type="file" name="myFile" /><br /> + <input type="submit" /> + </form> + <h2>Download a file</h2> + <a href='download'>This one</a> + </body></html> + """ + index.exposed = True + + def upload(self, myFile): + out = """<html> + <body> + myFile length: %s<br /> + myFile filename: %s<br /> + myFile mime-type: %s + </body> + </html>""" + + # Although this just counts the file length, it demonstrates + # how to read large files in chunks instead of all at once. + # CherryPy reads the uploaded file into a temporary file; + # myFile.file.read reads from that. + size = 0 + while True: + data = myFile.file.read(8192) + if not data: + break + size += len(data) + + return out % (size, myFile.filename, myFile.content_type) + upload.exposed = True + + def download(self): + path = os.path.join(absDir, "pdf_file.pdf") + return static.serve_file(path, "application/x-download", + "attachment", os.path.basename(path)) + download.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(FileDemo(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(FileDemo(), config=tutconf) diff --git a/cherrypy/tutorial/tut10_http_errors.py b/cherrypy/tutorial/tut10_http_errors.py new file mode 100755 index 0000000..dfa5733 --- /dev/null +++ b/cherrypy/tutorial/tut10_http_errors.py @@ -0,0 +1,81 @@ +""" + +Tutorial: HTTP errors + +HTTPError is used to return an error response to the client. +CherryPy has lots of options regarding how such errors are +logged, displayed, and formatted. + +""" + +import os +localDir = os.path.dirname(__file__) +curpath = os.path.normpath(os.path.join(os.getcwd(), localDir)) + +import cherrypy + + +class HTTPErrorDemo(object): + + # Set a custom response for 403 errors. + _cp_config = {'error_page.403' : os.path.join(curpath, "custom_error.html")} + + def index(self): + # display some links that will result in errors + tracebacks = cherrypy.request.show_tracebacks + if tracebacks: + trace = 'off' + else: + trace = 'on' + + return """ + <html><body> + <p>Toggle tracebacks <a href="toggleTracebacks">%s</a></p> + <p><a href="/doesNotExist">Click me; I'm a broken link!</a></p> + <p><a href="/error?code=403">Use a custom error page from a file.</a></p> + <p>These errors are explicitly raised by the application:</p> + <ul> + <li><a href="/error?code=400">400</a></li> + <li><a href="/error?code=401">401</a></li> + <li><a href="/error?code=402">402</a></li> + <li><a href="/error?code=500">500</a></li> + </ul> + <p><a href="/messageArg">You can also set the response body + when you raise an error.</a></p> + </body></html> + """ % trace + index.exposed = True + + def toggleTracebacks(self): + # simple function to toggle tracebacks on and off + tracebacks = cherrypy.request.show_tracebacks + cherrypy.config.update({'request.show_tracebacks': not tracebacks}) + + # redirect back to the index + raise cherrypy.HTTPRedirect('/') + toggleTracebacks.exposed = True + + def error(self, code): + # raise an error based on the get query + raise cherrypy.HTTPError(status = code) + error.exposed = True + + def messageArg(self): + message = ("If you construct an HTTPError with a 'message' " + "argument, it wil be placed on the error page " + "(underneath the status line by default).") + raise cherrypy.HTTPError(500, message=message) + messageArg.exposed = True + + +import os.path +tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf') + +if __name__ == '__main__': + # CherryPy always starts with app.root when trying to map request URIs + # to objects, so we need to mount a request handler root. A request + # to '/' will be mapped to HelloWorld().index(). + cherrypy.quickstart(HTTPErrorDemo(), config=tutconf) +else: + # This branch is for the test suite; you can ignore it. + cherrypy.tree.mount(HTTPErrorDemo(), config=tutconf) diff --git a/cherrypy/tutorial/tutorial.conf b/cherrypy/tutorial/tutorial.conf new file mode 100644 index 0000000..6537fd3 --- /dev/null +++ b/cherrypy/tutorial/tutorial.conf @@ -0,0 +1,4 @@ +[global]
+server.socket_host = "127.0.0.1"
+server.socket_port = 8080
+server.thread_pool = 10
diff --git a/cherrypy/wsgiserver/__init__.py b/cherrypy/wsgiserver/__init__.py new file mode 100755 index 0000000..55d1dd9 --- /dev/null +++ b/cherrypy/wsgiserver/__init__.py @@ -0,0 +1,2219 @@ +"""A high-speed, production ready, thread pooled, generic HTTP server. + +Simplest example on how to use this module directly +(without using CherryPy's application machinery):: + + from cherrypy import wsgiserver + + def my_crazy_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type','text/plain')] + start_response(status, response_headers) + return ['Hello world!'] + + server = wsgiserver.CherryPyWSGIServer( + ('0.0.0.0', 8070), my_crazy_app, + server_name='www.cherrypy.example') + server.start() + +The CherryPy WSGI server can serve as many WSGI applications +as you want in one instance by using a WSGIPathInfoDispatcher:: + + d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) + server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) + +Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. + +This won't call the CherryPy engine (application side) at all, only the +HTTP server, which is independent from the rest of CherryPy. Don't +let the name "CherryPyWSGIServer" throw you; the name merely reflects +its origin, not its coupling. + +For those of you wanting to understand internals of this module, here's the +basic call flow. The server's listening thread runs a very tight loop, +sticking incoming connections onto a Queue:: + + server = CherryPyWSGIServer(...) + server.start() + while True: + tick() + # This blocks until a request comes in: + child = socket.accept() + conn = HTTPConnection(child, ...) + server.requests.put(conn) + +Worker threads are kept in a pool and poll the Queue, popping off and then +handling each connection in turn. Each connection can consist of an arbitrary +number of requests and their responses, so we run a nested loop:: + + while True: + conn = server.requests.get() + conn.communicate() + -> while True: + req = HTTPRequest(...) + req.parse_request() + -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" + req.rfile.readline() + read_headers(req.rfile, req.inheaders) + req.respond() + -> response = app(...) + try: + for chunk in response: + if chunk: + req.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + if req.close_connection: + return +""" + +CRLF = '\r\n' +import os +import Queue +import re +quoted_slash = re.compile("(?i)%2F") +import rfc822 +import socket +import sys +if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 +try: + import cStringIO as StringIO +except ImportError: + import StringIO +DEFAULT_BUFFER_SIZE = -1 + +_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) + +import threading +import time +import traceback +def format_exc(limit=None): + """Like print_exc() but return a string. Backport for Python 2.3.""" + try: + etype, value, tb = sys.exc_info() + return ''.join(traceback.format_exception(etype, value, tb, limit)) + finally: + etype = value = tb = None + + +from urllib import unquote +from urlparse import urlparse +import warnings + +import errno + +def plat_specific_errors(*errnames): + """Return error numbers for all errors in errnames on this platform. + + The 'errno' module contains different global constants depending on + the specific platform (OS). This function will return the list of + numeric values for a given list of potential names. + """ + errno_names = dir(errno) + nums = [getattr(errno, k) for k in errnames if k in errno_names] + # de-dupe the list + return dict.fromkeys(nums).keys() + +socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") + +socket_errors_to_ignore = plat_specific_errors( + "EPIPE", + "EBADF", "WSAEBADF", + "ENOTSOCK", "WSAENOTSOCK", + "ETIMEDOUT", "WSAETIMEDOUT", + "ECONNREFUSED", "WSAECONNREFUSED", + "ECONNRESET", "WSAECONNRESET", + "ECONNABORTED", "WSAECONNABORTED", + "ENETRESET", "WSAENETRESET", + "EHOSTDOWN", "EHOSTUNREACH", + ) +socket_errors_to_ignore.append("timed out") +socket_errors_to_ignore.append("The read operation timed out") + +socket_errors_nonblocking = plat_specific_errors( + 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') + +comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', + 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', + 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', + 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', + 'WWW-Authenticate'] + + +import logging +if not hasattr(logging, 'statistics'): logging.statistics = {} + + +def read_headers(rfile, hdict=None): + """Read headers from the given stream into the given header dict. + + If hdict is None, a new header dict is created. Returns the populated + header dict. + + Headers which are repeated are folded together using a comma if their + specification so dictates. + + This function raises ValueError when the read bytes violate the HTTP spec. + You should probably return "400 Bad Request" if this happens. + """ + if hdict is None: + hdict = {} + + while True: + line = rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + if line[0] in ' \t': + # It's a continuation line. + v = line.strip() + else: + try: + k, v = line.split(":", 1) + except ValueError: + raise ValueError("Illegal header line.") + # TODO: what about TE and WWW-Authenticate? + k = k.strip().title() + v = v.strip() + hname = k + + if k in comma_separated_headers: + existing = hdict.get(hname) + if existing: + v = ", ".join((existing, v)) + hdict[hname] = v + + return hdict + + +class MaxSizeExceeded(Exception): + pass + +class SizeCheckWrapper(object): + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" + + def __init__(self, rfile, maxlen): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + + def _check_length(self): + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded() + + def read(self, size=None): + data = self.rfile.read(size) + self.bytes_read += len(data) + self._check_length() + return data + + def readline(self, size=None): + if size is not None: + data = self.rfile.readline(size) + self.bytes_read += len(data) + self._check_length() + return data + + # User didn't specify a size ... + # We read the line in chunks to make sure it's not a 100MB line ! + res = [] + while True: + data = self.rfile.readline(256) + self.bytes_read += len(data) + self._check_length() + res.append(data) + # See http://www.cherrypy.org/ticket/421 + if len(data) < 256 or data[-1:] == "\n": + return ''.join(res) + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline() + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline() + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def next(self): + data = self.rfile.next() + self.bytes_read += len(data) + self._check_length() + return data + + +class KnownLengthRFile(object): + """Wraps a file-like object, returning an empty string when exhausted.""" + + def __init__(self, rfile, content_length): + self.rfile = rfile + self.remaining = content_length + + def read(self, size=None): + if self.remaining == 0: + return '' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.read(size) + self.remaining -= len(data) + return data + + def readline(self, size=None): + if self.remaining == 0: + return '' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.readline(size) + self.remaining -= len(data) + return data + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.remaining -= len(data) + return data + + +class ChunkedRFile(object): + """Wraps a file-like object, returning an empty string when exhausted. + + This class is intended to provide a conforming wsgi.input value for + request entities that have been encoded with the 'chunked' transfer + encoding. + """ + + def __init__(self, rfile, maxlen, bufsize=8192): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + self.buffer = '' + self.bufsize = bufsize + self.closed = False + + def _fetch(self): + if self.closed: + return + + line = self.rfile.readline() + self.bytes_read += len(line) + + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) + + line = line.strip().split(";", 1) + + try: + chunk_size = line.pop(0) + chunk_size = int(chunk_size, 16) + except ValueError: + raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) + + if chunk_size <= 0: + self.closed = True + return + +## if line: chunk_extension = line[0] + + if self.maxlen and self.bytes_read + chunk_size > self.maxlen: + raise IOError("Request Entity Too Large") + + chunk = self.rfile.read(chunk_size) + self.bytes_read += len(chunk) + self.buffer += chunk + + crlf = self.rfile.read(2) + if crlf != CRLF: + raise ValueError( + "Bad chunked transfer coding (expected '\\r\\n', " + "got " + repr(crlf) + ")") + + def read(self, size=None): + data = '' + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + if size: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + data += self.buffer + + def readline(self, size=None): + data = '' + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + newline_pos = self.buffer.find('\n') + if size: + if newline_pos == -1: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + remaining = min(size - len(data), newline_pos) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + if newline_pos == -1: + data += self.buffer + else: + data += self.buffer[:newline_pos] + self.buffer = self.buffer[newline_pos:] + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def read_trailer_lines(self): + if not self.closed: + raise ValueError( + "Cannot read trailers until the request body has been read.") + + while True: + line = self.rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + self.bytes_read += len(line) + if self.maxlen and self.bytes_read > self.maxlen: + raise IOError("Request Entity Too Large") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + yield line + + def close(self): + self.rfile.close() + + def __iter__(self): + # Shamelessly stolen from StringIO + total = 0 + line = self.readline(sizehint) + while line: + yield line + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + + +class HTTPRequest(object): + """An HTTP Request (and response). + + A single HTTP connection may consist of multiple request/response pairs. + """ + + server = None + """The HTTPServer object which is receiving this request.""" + + conn = None + """The HTTPConnection object on which this request connected.""" + + inheaders = {} + """A dict of request headers.""" + + outheaders = [] + """A list of header tuples to write in the response.""" + + ready = False + """When True, the request has been parsed and is ready to begin generating + the response. When False, signals the calling Connection that the response + should not be generated and the connection should close.""" + + close_connection = False + """Signals the calling Connection that the request should close. This does + not imply an error! The client and/or server may each request that the + connection be closed.""" + + chunked_write = False + """If True, output will be encoded with the "chunked" transfer-coding. + + This value is set automatically inside send_headers.""" + + def __init__(self, server, conn): + self.server= server + self.conn = conn + + self.ready = False + self.started_request = False + self.scheme = "http" + if self.server.ssl_adapter is not None: + self.scheme = "https" + # Use the lowest-common protocol in case read_request_line errors. + self.response_protocol = 'HTTP/1.0' + self.inheaders = {} + + self.status = "" + self.outheaders = [] + self.sent_headers = False + self.close_connection = self.__class__.close_connection + self.chunked_read = False + self.chunked_write = self.__class__.chunked_write + + def parse_request(self): + """Parse the next HTTP request start-line and message-headers.""" + self.rfile = SizeCheckWrapper(self.conn.rfile, + self.server.max_request_header_size) + try: + self.read_request_line() + except MaxSizeExceeded: + self.simple_response("414 Request-URI Too Long", + "The Request-URI sent with the request exceeds the maximum " + "allowed bytes.") + return + + try: + success = self.read_request_headers() + except MaxSizeExceeded: + self.simple_response("413 Request Entity Too Large", + "The headers sent with the request exceed the maximum " + "allowed bytes.") + return + else: + if not success: + return + + self.ready = True + + def read_request_line(self): + # HTTP/1.1 connections are persistent by default. If a client + # requests a page, then idles (leaves the connection open), + # then rfile.readline() will raise socket.error("timed out"). + # Note that it does this based on the value given to settimeout(), + # and doesn't need the client to request or acknowledge the close + # (although your TCP stack might suffer for it: cf Apache's history + # with FIN_WAIT_2). + request_line = self.rfile.readline() + + # Set started_request to True so communicate() knows to send 408 + # from here on out. + self.started_request = True + if not request_line: + # Force self.ready = False so the connection will close. + self.ready = False + return + + if request_line == CRLF: + # RFC 2616 sec 4.1: "...if the server is reading the protocol + # stream at the beginning of a message and receives a CRLF + # first, it should ignore the CRLF." + # But only ignore one leading line! else we enable a DoS. + request_line = self.rfile.readline() + if not request_line: + self.ready = False + return + + if not request_line.endswith(CRLF): + self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") + return + + try: + method, uri, req_protocol = request_line.strip().split(" ", 2) + rp = int(req_protocol[5]), int(req_protocol[7]) + except (ValueError, IndexError): + self.simple_response("400 Bad Request", "Malformed Request-Line") + return + + self.uri = uri + self.method = method + + # uri may be an abs_path (including "http://host.domain.tld"); + scheme, authority, path = self.parse_request_uri(uri) + if '#' in path: + self.simple_response("400 Bad Request", + "Illegal #fragment in Request-URI.") + return + + if scheme: + self.scheme = scheme + + qs = '' + if '?' in path: + path, qs = path.split('?', 1) + + # Unquote the path+params (e.g. "/this%20path" -> "/this path"). + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". + try: + atoms = [unquote(x) for x in quoted_slash.split(path)] + except ValueError, ex: + self.simple_response("400 Bad Request", ex.args[0]) + return + path = "%2F".join(atoms) + self.path = path + + # Note that, like wsgiref and most other HTTP servers, + # we "% HEX HEX"-unquote the path but not the query string. + self.qs = qs + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + sp = int(self.server.protocol[5]), int(self.server.protocol[7]) + + if sp[0] != rp[0]: + self.simple_response("505 HTTP Version Not Supported") + return + self.request_protocol = req_protocol + self.response_protocol = "HTTP/%s.%s" % min(rp, sp) + + def read_request_headers(self): + """Read self.rfile into self.inheaders. Return success.""" + + # then all the http headers + try: + read_headers(self.rfile, self.inheaders) + except ValueError, ex: + self.simple_response("400 Bad Request", ex.args[0]) + return False + + mrbs = self.server.max_request_body_size + if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return False + + # Persistent connection support + if self.response_protocol == "HTTP/1.1": + # Both server and client are HTTP/1.1 + if self.inheaders.get("Connection", "") == "close": + self.close_connection = True + else: + # Either the server or client (or both) are HTTP/1.0 + if self.inheaders.get("Connection", "") != "Keep-Alive": + self.close_connection = True + + # Transfer-Encoding support + te = None + if self.response_protocol == "HTTP/1.1": + te = self.inheaders.get("Transfer-Encoding") + if te: + te = [x.strip().lower() for x in te.split(",") if x.strip()] + + self.chunked_read = False + + if te: + for enc in te: + if enc == "chunked": + self.chunked_read = True + else: + # Note that, even if we see "chunked", we must reject + # if there is an extension we don't recognize. + self.simple_response("501 Unimplemented") + self.close_connection = True + return False + + # From PEP 333: + # "Servers and gateways that implement HTTP 1.1 must provide + # transparent support for HTTP 1.1's "expect/continue" mechanism. + # This may be done in any of several ways: + # 1. Respond to requests containing an Expect: 100-continue request + # with an immediate "100 Continue" response, and proceed normally. + # 2. Proceed with the request normally, but provide the application + # with a wsgi.input stream that will send the "100 Continue" + # response if/when the application first attempts to read from + # the input stream. The read request must then remain blocked + # until the client responds. + # 3. Wait until the client decides that the server does not support + # expect/continue, and sends the request body on its own. + # (This is suboptimal, and is not recommended.) + # + # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, + # but it seems like it would be a big slowdown for such a rare case. + if self.inheaders.get("Expect", "") == "100-continue": + # Don't use simple_response here, because it emits headers + # we don't want. See http://www.cherrypy.org/ticket/951 + msg = self.server.protocol + " 100 Continue\r\n\r\n" + try: + self.conn.wfile.sendall(msg) + except socket.error, x: + if x.args[0] not in socket_errors_to_ignore: + raise + return True + + def parse_request_uri(self, uri): + """Parse a Request-URI into (scheme, authority, path). + + Note that Request-URI's must be one of:: + + Request-URI = "*" | absoluteURI | abs_path | authority + + Therefore, a Request-URI which starts with a double forward-slash + cannot be a "net_path":: + + net_path = "//" authority [ abs_path ] + + Instead, it must be interpreted as an "abs_path" with an empty first + path segment:: + + abs_path = "/" path_segments + path_segments = segment *( "/" segment ) + segment = *pchar *( ";" param ) + param = *pchar + """ + if uri == "*": + return None, None, uri + + i = uri.find('://') + if i > 0 and '?' not in uri[:i]: + # An absoluteURI. + # If there's a scheme (and it must be http or https), then: + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + scheme, remainder = uri[:i].lower(), uri[i + 3:] + authority, path = remainder.split("/", 1) + return scheme, authority, path + + if uri.startswith('/'): + # An abs_path. + return None, None, uri + else: + # An authority. + return None, uri, None + + def respond(self): + """Call the gateway and write its iterable output.""" + mrbs = self.server.max_request_body_size + if self.chunked_read: + self.rfile = ChunkedRFile(self.conn.rfile, mrbs) + else: + cl = int(self.inheaders.get("Content-Length", 0)) + if mrbs and mrbs < cl: + if not self.sent_headers: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return + self.rfile = KnownLengthRFile(self.conn.rfile, cl) + + self.server.gateway(self).respond() + + if (self.ready and not self.sent_headers): + self.sent_headers = True + self.send_headers() + if self.chunked_write: + self.conn.wfile.sendall("0\r\n\r\n") + + def simple_response(self, status, msg=""): + """Write a simple response back to the client.""" + status = str(status) + buf = [self.server.protocol + " " + + status + CRLF, + "Content-Length: %s\r\n" % len(msg), + "Content-Type: text/plain\r\n"] + + if status[:3] in ("413", "414"): + # Request Entity Too Large / Request-URI Too Long + self.close_connection = True + if self.response_protocol == 'HTTP/1.1': + # This will not be true for 414, since read_request_line + # usually raises 414 before reading the whole line, and we + # therefore cannot know the proper response_protocol. + buf.append("Connection: close\r\n") + else: + # HTTP/1.0 had no 413/414 status nor Connection header. + # Emit 400 instead and trust the message body is enough. + status = "400 Bad Request" + + buf.append(CRLF) + if msg: + if isinstance(msg, unicode): + msg = msg.encode("ISO-8859-1") + buf.append(msg) + + try: + self.conn.wfile.sendall("".join(buf)) + except socket.error, x: + if x.args[0] not in socket_errors_to_ignore: + raise + + def write(self, chunk): + """Write unbuffered data to the client.""" + if self.chunked_write and chunk: + buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF] + self.conn.wfile.sendall("".join(buf)) + else: + self.conn.wfile.sendall(chunk) + + def send_headers(self): + """Assert, process, and send the HTTP response message-headers. + + You must set self.status, and self.outheaders before calling this. + """ + hkeys = [key.lower() for key, value in self.outheaders] + status = int(self.status[:3]) + + if status == 413: + # Request Entity Too Large. Close conn to avoid garbage. + self.close_connection = True + elif "content-length" not in hkeys: + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." So no point chunking. + if status < 200 or status in (204, 205, 304): + pass + else: + if (self.response_protocol == 'HTTP/1.1' + and self.method != 'HEAD'): + # Use the chunked transfer-coding + self.chunked_write = True + self.outheaders.append(("Transfer-Encoding", "chunked")) + else: + # Closing the conn is the only way to determine len. + self.close_connection = True + + if "connection" not in hkeys: + if self.response_protocol == 'HTTP/1.1': + # Both server and client are HTTP/1.1 or better + if self.close_connection: + self.outheaders.append(("Connection", "close")) + else: + # Server and/or client are HTTP/1.0 + if not self.close_connection: + self.outheaders.append(("Connection", "Keep-Alive")) + + if (not self.close_connection) and (not self.chunked_read): + # Read any remaining request body data on the socket. + # "If an origin server receives a request that does not include an + # Expect request-header field with the "100-continue" expectation, + # the request includes a request body, and the server responds + # with a final status code before reading the entire request body + # from the transport connection, then the server SHOULD NOT close + # the transport connection until it has read the entire request, + # or until the client closes the connection. Otherwise, the client + # might not reliably receive the response message. However, this + # requirement is not be construed as preventing a server from + # defending itself against denial-of-service attacks, or from + # badly broken client implementations." + remaining = getattr(self.rfile, 'remaining', 0) + if remaining > 0: + self.rfile.read(remaining) + + if "date" not in hkeys: + self.outheaders.append(("Date", rfc822.formatdate())) + + if "server" not in hkeys: + self.outheaders.append(("Server", self.server.server_name)) + + buf = [self.server.protocol + " " + self.status + CRLF] + for k, v in self.outheaders: + buf.append(k + ": " + v + CRLF) + buf.append(CRLF) + self.conn.wfile.sendall("".join(buf)) + + +class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" + pass + + +class FatalSSLAlert(Exception): + """Exception raised when the SSL implementation signals a fatal alert.""" + pass + + +class CP_fileobject(socket._fileobject): + """Faux file object attached to a socket object.""" + + def __init__(self, *args, **kwargs): + self.bytes_read = 0 + self.bytes_written = 0 + socket._fileobject.__init__(self, *args, **kwargs) + + def sendall(self, data): + """Sendall for non-blocking sockets.""" + while data: + try: + bytes_sent = self.send(data) + data = data[bytes_sent:] + except socket.error, e: + if e.args[0] not in socket_errors_nonblocking: + raise + + def send(self, data): + bytes_sent = self._sock.send(data) + self.bytes_written += bytes_sent + return bytes_sent + + def flush(self): + if self._wbuf: + buffer = "".join(self._wbuf) + self._wbuf = [] + self.sendall(buffer) + + def recv(self, size): + while True: + try: + data = self._sock.recv(size) + self.bytes_read += len(data) + return data + except socket.error, e: + if (e.args[0] not in socket_errors_nonblocking + and e.args[0] not in socket_error_eintr): + raise + + if not _fileobject_uses_str_type: + def read(self, size=-1): + # Use max, disallow tiny reads in a loop as they are very inefficient. + # We never leave read() with any leftover data from a new recv() call + # in our internal buffer. + rbufsize = max(self._rbufsize, self.default_bufsize) + # Our use of StringIO rather than lists of string objects returned by + # recv() minimizes memory usage and fragmentation that occurs when + # rbufsize is large compared to the typical return value of recv(). + buf = self._rbuf + buf.seek(0, 2) # seek end + if size < 0: + # Read until EOF + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(rbufsize) + if not data: + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or EOF seen, whichever comes first + buf_len = buf.tell() + if buf_len >= size: + # Already have size bytes in our buffer? Extract and return. + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return rv + + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + left = size - buf_len + # recv() will malloc the amount of memory given as its + # parameter even though it often returns much less data + # than that. The returned data string is short lived + # as we copy it into a StringIO and free it. This avoids + # fragmentation issues on many platforms. + data = self.recv(left) + if not data: + break + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid buffer data copies when: + # - We have no data in our buffer. + # AND + # - Our call to recv returned exactly the + # number of bytes we were asked to read. + return data + if n == left: + buf.write(data) + del data # explicit free + break + assert n <= left, "recv(%d) returned %d bytes" % (left, n) + buf.write(data) + buf_len += n + del data # explicit free + #assert buf_len == buf.tell() + return buf.getvalue() + + def readline(self, size=-1): + buf = self._rbuf + buf.seek(0, 2) # seek end + if buf.tell() > 0: + # check if we already have it in our buffer + buf.seek(0) + bline = buf.readline(size) + if bline.endswith('\n') or len(bline) == size: + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return bline + del bline + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + buf.seek(0) + buffers = [buf.read()] + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + data = None + recv = self.recv + while data != "\n": + data = recv(1) + if not data: + break + buffers.append(data) + return "".join(buffers) + + buf.seek(0, 2) # seek end + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(self._rbufsize) + if not data: + break + nl = data.find('\n') + if nl >= 0: + nl += 1 + buf.write(data[:nl]) + self._rbuf.write(data[nl:]) + del data + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or \n or EOF seen, whichever comes first + buf.seek(0, 2) # seek end + buf_len = buf.tell() + if buf_len >= size: + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return rv + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(self._rbufsize) + if not data: + break + left = size - buf_len + # did we just receive a newline? + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + # save the excess data to _rbuf + self._rbuf.write(data[nl:]) + if buf_len: + buf.write(data[:nl]) + break + else: + # Shortcut. Avoid data copy through buf when returning + # a substring of our first recv(). + return data[:nl] + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid data copy through buf when + # returning exactly all of our first recv(). + return data + if n >= left: + buf.write(data[:left]) + self._rbuf.write(data[left:]) + break + buf.write(data) + buf_len += n + #assert buf_len == buf.tell() + return buf.getvalue() + else: + def read(self, size=-1): + if size < 0: + # Read until EOF + buffers = [self._rbuf] + self._rbuf = "" + if self._rbufsize <= 1: + recv_size = self.default_bufsize + else: + recv_size = self._rbufsize + + while True: + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + return "".join(buffers) + else: + # Read until size bytes or EOF seen, whichever comes first + data = self._rbuf + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + left = size - buf_len + recv_size = max(self._rbufsize, left) + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return "".join(buffers) + + def readline(self, size=-1): + data = self._rbuf + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + assert data == "" + buffers = [] + while data != "\n": + data = self.recv(1) + if not data: + break + buffers.append(data) + return "".join(buffers) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + return "".join(buffers) + else: + # Read until size bytes or \n or EOF seen, whichever comes first + nl = data.find('\n', 0, size) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + left = size - buf_len + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return "".join(buffers) + + +class HTTPConnection(object): + """An HTTP connection (active socket). + + server: the Server object which received this connection. + socket: the raw socket object (usually TCP) for this connection. + makefile: a fileobject class for reading from the socket. + """ + + remote_addr = None + remote_port = None + ssl_env = None + rbufsize = DEFAULT_BUFFER_SIZE + wbufsize = DEFAULT_BUFFER_SIZE + RequestHandlerClass = HTTPRequest + + def __init__(self, server, sock, makefile=CP_fileobject): + self.server = server + self.socket = sock + self.rfile = makefile(sock, "rb", self.rbufsize) + self.wfile = makefile(sock, "wb", self.wbufsize) + self.requests_seen = 0 + + def communicate(self): + """Read each request and respond appropriately.""" + request_seen = False + try: + while True: + # (re)set req to None so that if something goes wrong in + # the RequestHandlerClass constructor, the error doesn't + # get written to the previous request. + req = None + req = self.RequestHandlerClass(self.server, self) + + # This order of operations should guarantee correct pipelining. + req.parse_request() + if self.server.stats['Enabled']: + self.requests_seen += 1 + if not req.ready: + # Something went wrong in the parsing (and the server has + # probably already made a simple_response). Return and + # let the conn close. + return + + request_seen = True + req.respond() + if req.close_connection: + return + except socket.error, e: + errnum = e.args[0] + # sadly SSL sockets return a different (longer) time out string + if errnum == 'timed out' or errnum == 'The read operation timed out': + # Don't error if we're between requests; only error + # if 1) no request has been started at all, or 2) we're + # in the middle of a request. + # See http://www.cherrypy.org/ticket/853 + if (not request_seen) or (req and req.started_request): + # Don't bother writing the 408 if the response + # has already started being written. + if req and not req.sent_headers: + try: + req.simple_response("408 Request Timeout") + except FatalSSLAlert: + # Close the connection. + return + elif errnum not in socket_errors_to_ignore: + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error", + format_exc()) + except FatalSSLAlert: + # Close the connection. + return + return + except (KeyboardInterrupt, SystemExit): + raise + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + if req and not req.sent_headers: + # Unwrap our wfile + self.wfile = CP_fileobject(self.socket._sock, "wb", self.wbufsize) + req.simple_response("400 Bad Request", + "The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + self.linger = True + except Exception: + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error", format_exc()) + except FatalSSLAlert: + # Close the connection. + return + + linger = False + + def close(self): + """Close the socket underlying this connection.""" + self.rfile.close() + + if not self.linger: + # Python's socket module does NOT call close on the kernel socket + # when you call socket.close(). We do so manually here because we + # want this server to send a FIN TCP segment immediately. Note this + # must be called *before* calling socket.close(), because the latter + # drops its reference to the kernel socket. + if hasattr(self.socket, '_sock'): + self.socket._sock.close() + self.socket.close() + else: + # On the other hand, sometimes we want to hang around for a bit + # to make sure the client has a chance to read our entire + # response. Skipping the close() calls here delays the FIN + # packet until the socket object is garbage-collected later. + # Someday, perhaps, we'll do the full lingering_close that + # Apache does, but not today. + pass + + +_SHUTDOWNREQUEST = None + +class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. + + Due to the timing issues of polling a Queue, a WorkerThread does not + check its own 'ready' flag after it has started. To stop the thread, + it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue + (one for each running WorkerThread). + """ + + conn = None + """The current connection pulled off the Queue, or None.""" + + server = None + """The HTTP Server which spawned this thread, and which owns the + Queue and is placing active connections into it.""" + + ready = False + """A simple flag for the calling server to know when this thread + has begun polling the Queue.""" + + + def __init__(self, server): + self.ready = False + self.server = server + + self.requests_seen = 0 + self.bytes_read = 0 + self.bytes_written = 0 + self.start_time = None + self.work_time = 0 + self.stats = { + 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and 0 or self.conn.requests_seen), + 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and 0 or self.conn.rfile.bytes_read), + 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and 0 or self.conn.wfile.bytes_written), + 'Work Time': lambda s: self.work_time + ((self.start_time is None) and 0 or time.time() - self.start_time), + 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), + } + threading.Thread.__init__(self) + + def run(self): + self.server.stats['Worker Threads'][self.getName()] = self.stats + try: + self.ready = True + while True: + conn = self.server.requests.get() + if conn is _SHUTDOWNREQUEST: + return + + self.conn = conn + if self.server.stats['Enabled']: + self.start_time = time.time() + try: + conn.communicate() + finally: + conn.close() + if self.server.stats['Enabled']: + self.requests_seen += self.conn.requests_seen + self.bytes_read += self.conn.rfile.bytes_read + self.bytes_written += self.conn.wfile.bytes_written + self.work_time += time.time() - self.start_time + self.start_time = None + self.conn = None + except (KeyboardInterrupt, SystemExit), exc: + self.server.interrupt = exc + + +class ThreadPool(object): + """A Request Queue for the CherryPyWSGIServer which pools threads. + + ThreadPool objects must provide min, get(), put(obj), start() + and stop(timeout) attributes. + """ + + def __init__(self, server, min=10, max=-1): + self.server = server + self.min = min + self.max = max + self._threads = [] + self._queue = Queue.Queue() + self.get = self._queue.get + + def start(self): + """Start the pool of threads.""" + for i in range(self.min): + self._threads.append(WorkerThread(self.server)) + for worker in self._threads: + worker.setName("CP Server " + worker.getName()) + worker.start() + for worker in self._threads: + while not worker.ready: + time.sleep(.1) + + def _get_idle(self): + """Number of worker threads which are idle. Read-only.""" + return len([t for t in self._threads if t.conn is None]) + idle = property(_get_idle, doc=_get_idle.__doc__) + + def put(self, obj): + self._queue.put(obj) + if obj is _SHUTDOWNREQUEST: + return + + def grow(self, amount): + """Spawn new worker threads (not above self.max).""" + for i in range(amount): + if self.max > 0 and len(self._threads) >= self.max: + break + worker = WorkerThread(self.server) + worker.setName("CP Server " + worker.getName()) + self._threads.append(worker) + worker.start() + + def shrink(self, amount): + """Kill off worker threads (not below self.min).""" + # Grow/shrink the pool if necessary. + # Remove any dead threads from our list + for t in self._threads: + if not t.isAlive(): + self._threads.remove(t) + amount -= 1 + + if amount > 0: + for i in range(min(amount, len(self._threads) - self.min)): + # Put a number of shutdown requests on the queue equal + # to 'amount'. Once each of those is processed by a worker, + # that worker will terminate and be culled from our list + # in self.put. + self._queue.put(_SHUTDOWNREQUEST) + + def stop(self, timeout=5): + # Must shut down threads here so the code that calls + # this method can know when all threads are stopped. + for worker in self._threads: + self._queue.put(_SHUTDOWNREQUEST) + + # Don't join currentThread (when stop is called inside a request). + current = threading.currentThread() + if timeout and timeout >= 0: + endtime = time.time() + timeout + while self._threads: + worker = self._threads.pop() + if worker is not current and worker.isAlive(): + try: + if timeout is None or timeout < 0: + worker.join() + else: + remaining_time = endtime - time.time() + if remaining_time > 0: + worker.join(remaining_time) + if worker.isAlive(): + # We exhausted the timeout. + # Forcibly shut down the socket. + c = worker.conn + if c and not c.rfile.closed: + try: + c.socket.shutdown(socket.SHUT_RD) + except TypeError: + # pyOpenSSL sockets don't take an arg + c.socket.shutdown() + worker.join() + except (AssertionError, + # Ignore repeated Ctrl-C. + # See http://www.cherrypy.org/ticket/691. + KeyboardInterrupt), exc1: + pass + + def _get_qsize(self): + return self._queue.qsize() + qsize = property(_get_qsize) + + + +try: + import fcntl +except ImportError: + try: + from ctypes import windll, WinError + except ImportError: + def prevent_socket_inheritance(sock): + """Dummy function, since neither fcntl nor ctypes are available.""" + pass + else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (Windows).""" + if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): + raise WinError() +else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (POSIX).""" + fd = sock.fileno() + old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) + + +class SSLAdapter(object): + """Base class for SSL driver library adapters. + + Required methods: + + * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` + """ + + def __init__(self, certificate, private_key, certificate_chain=None): + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + + def wrap(self, sock): + raise NotImplemented + + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + raise NotImplemented + + +class HTTPServer(object): + """An HTTP server.""" + + _bind_addr = "127.0.0.1" + _interrupt = None + + gateway = None + """A Gateway instance.""" + + minthreads = None + """The minimum number of worker threads to create (default 10).""" + + maxthreads = None + """The maximum number of worker threads to create (default -1 = no limit).""" + + server_name = None + """The name of the server; defaults to socket.gethostname().""" + + protocol = "HTTP/1.1" + """The version string to write in the Status-Line of all HTTP responses. + + For example, "HTTP/1.1" is the default. This also limits the supported + features used in the response.""" + + request_queue_size = 5 + """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + + shutdown_timeout = 5 + """The total time, in seconds, to wait for worker threads to cleanly exit.""" + + timeout = 10 + """The timeout in seconds for accepted connections (default 10).""" + + version = "CherryPy/3.2.0" + """A version string for the HTTPServer.""" + + software = None + """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. + + If None, this defaults to ``'%s Server' % self.version``.""" + + ready = False + """An internal flag which marks whether the socket is accepting connections.""" + + max_request_header_size = 0 + """The maximum size, in bytes, for request headers, or 0 for no limit.""" + + max_request_body_size = 0 + """The maximum size, in bytes, for request bodies, or 0 for no limit.""" + + nodelay = True + """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" + + ConnectionClass = HTTPConnection + """The class to use for handling HTTP connections.""" + + ssl_adapter = None + """An instance of SSLAdapter (or a subclass). + + You must have the corresponding SSL driver library installed.""" + + def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, + server_name=None): + self.bind_addr = bind_addr + self.gateway = gateway + + self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) + + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.clear_stats() + + def clear_stats(self): + self._start_time = None + self._run_time = 0 + self.stats = { + 'Enabled': False, + 'Bind Address': lambda s: repr(self.bind_addr), + 'Run time': lambda s: (not s['Enabled']) and 0 or self.runtime(), + 'Accepts': 0, + 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), + 'Queue': lambda s: getattr(self.requests, "qsize", None), + 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), + 'Threads Idle': lambda s: getattr(self.requests, "idle", None), + 'Socket Errors': 0, + 'Requests': lambda s: (not s['Enabled']) and 0 or sum([w['Requests'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and 0 or sum([w['Bytes Read'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and 0 or sum([w['Bytes Written'](w) for w + in s['Worker Threads'].values()], 0), + 'Work Time': lambda s: (not s['Enabled']) and 0 or sum([w['Work Time'](w) for w + in s['Worker Threads'].values()], 0), + 'Read Throughput': lambda s: (not s['Enabled']) and 0 or sum( + [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Write Throughput': lambda s: (not s['Enabled']) and 0 or sum( + [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Worker Threads': {}, + } + logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats + + def runtime(self): + if self._start_time is None: + return self._run_time + else: + return self._run_time + (time.time() - self._start_time) + + def __str__(self): + return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, + self.bind_addr) + + def _get_bind_addr(self): + return self._bind_addr + def _set_bind_addr(self, value): + if isinstance(value, tuple) and value[0] in ('', None): + # Despite the socket module docs, using '' does not + # allow AI_PASSIVE to work. Passing None instead + # returns '0.0.0.0' like we want. In other words: + # host AI_PASSIVE result + # '' Y 192.168.x.y + # '' N 192.168.x.y + # None Y 0.0.0.0 + # None N 127.0.0.1 + # But since you can get the same effect with an explicit + # '0.0.0.0', we deny both the empty string and None as values. + raise ValueError("Host values of '' or None are not allowed. " + "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " + "to listen on all active interfaces.") + self._bind_addr = value + bind_addr = property(_get_bind_addr, _set_bind_addr, + doc="""The interface on which to listen for connections. + + For TCP sockets, a (host, port) tuple. Host values may be any IPv4 + or IPv6 address, or any valid hostname. The string 'localhost' is a + synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). + The string '0.0.0.0' is a special IPv4 entry meaning "any active + interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for + IPv6. The empty string or None are not allowed. + + For UNIX sockets, supply the filename as a string.""") + + def start(self): + """Run the server forever.""" + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrpy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self._interrupt = None + + if self.software is None: + self.software = "%s Server" % self.version + + # SSL backward compatibility + if (self.ssl_adapter is None and + getattr(self, 'ssl_certificate', None) and + getattr(self, 'ssl_private_key', None)): + warnings.warn( + "SSL attributes are deprecated in CherryPy 3.2, and will " + "be removed in CherryPy 3.3. Use an ssl_adapter attribute " + "instead.", + DeprecationWarning + ) + try: + from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter + except ImportError: + pass + else: + self.ssl_adapter = pyOpenSSLAdapter( + self.ssl_certificate, self.ssl_private_key, + getattr(self, 'ssl_certificate_chain', None)) + + # Select the appropriate socket + if isinstance(self.bind_addr, basestring): + # AF_UNIX socket + + # So we can reuse the socket... + try: os.unlink(self.bind_addr) + except: pass + + # So everyone can access the socket... + try: os.chmod(self.bind_addr, 0777) + except: pass + + info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + else: + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 addresses) + host, port = self.bind_addr + try: + info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + except socket.gaierror: + if ':' in self.bind_addr[0]: + info = [(socket.AF_INET6, socket.SOCK_STREAM, + 0, "", self.bind_addr + (0, 0))] + else: + info = [(socket.AF_INET, socket.SOCK_STREAM, + 0, "", self.bind_addr)] + + self.socket = None + msg = "No socket could be created" + for res in info: + af, socktype, proto, canonname, sa = res + try: + self.bind(af, socktype, proto) + except socket.error: + if self.socket: + self.socket.close() + self.socket = None + continue + break + if not self.socket: + raise socket.error(msg) + + # Timeout so KeyboardInterrupt can be caught on Win32 + self.socket.settimeout(1) + self.socket.listen(self.request_queue_size) + + # Create worker threads + self.requests.start() + + self.ready = True + self._start_time = time.time() + while self.ready: + self.tick() + if self.interrupt: + while self.interrupt is True: + # Wait for self.stop() to complete. See _set_interrupt. + time.sleep(0.1) + if self.interrupt: + raise self.interrupt + + def bind(self, family, type, proto=0): + """Create (or recreate) the actual socket object.""" + self.socket = socket.socket(family, type, proto) + prevent_socket_inheritance(self.socket) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.nodelay and not isinstance(self.bind_addr, str): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if self.ssl_adapter is not None: + self.socket = self.ssl_adapter.bind(self.socket) + + # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), + # activate dual-stack. See http://www.cherrypy.org/ticket/871. + if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + try: + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + except (AttributeError, socket.error): + # Apparently, the socket option is not available in + # this machine's TCP stack + pass + + self.socket.bind(self.bind_addr) + + def tick(self): + """Accept a new connection and put it on the Queue.""" + try: + s, addr = self.socket.accept() + if self.stats['Enabled']: + self.stats['Accepts'] += 1 + if not self.ready: + return + + prevent_socket_inheritance(s) + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + makefile = CP_fileobject + ssl_env = {} + # if ssl cert and key are set, we try to be a secure HTTP server + if self.ssl_adapter is not None: + try: + s, ssl_env = self.ssl_adapter.wrap(s) + except NoSSLError: + msg = ("The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + buf = ["%s 400 Bad Request\r\n" % self.protocol, + "Content-Length: %s\r\n" % len(msg), + "Content-Type: text/plain\r\n\r\n", + msg] + + wfile = CP_fileobject(s, "wb", DEFAULT_BUFFER_SIZE) + try: + wfile.sendall("".join(buf)) + except socket.error, x: + if x.args[0] not in socket_errors_to_ignore: + raise + return + if not s: + return + makefile = self.ssl_adapter.makefile + # Re-apply our timeout since we may have a new socket object + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + conn = self.ConnectionClass(self, s, makefile) + + if not isinstance(self.bind_addr, basestring): + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + if addr is None: # sometimes this can happen + # figure out if AF_INET or AF_INET6. + if len(s.getsockname()) == 2: + # AF_INET + addr = ('0.0.0.0', 0) + else: + # AF_INET6 + addr = ('::', 0) + conn.remote_addr = addr[0] + conn.remote_port = addr[1] + + conn.ssl_env = ssl_env + + self.requests.put(conn) + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error, x: + if self.stats['Enabled']: + self.stats['Socket Errors'] += 1 + if x.args[0] in socket_error_eintr: + # I *think* this is right. EINTR should occur when a signal + # is received during the accept() call; all docs say retry + # the call, and I *think* I'm reading it right that Python + # will then go ahead and poll for and handle the signal + # elsewhere. See http://www.cherrypy.org/ticket/707. + return + if x.args[0] in socket_errors_nonblocking: + # Just try again. See http://www.cherrypy.org/ticket/479. + return + if x.args[0] in socket_errors_to_ignore: + # Our socket was closed. + # See http://www.cherrypy.org/ticket/686. + return + raise + + def _get_interrupt(self): + return self._interrupt + def _set_interrupt(self, interrupt): + self._interrupt = True + self.stop() + self._interrupt = interrupt + interrupt = property(_get_interrupt, _set_interrupt, + doc="Set this to an Exception instance to " + "interrupt the server.") + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + self.ready = False + if self._start_time is not None: + self._run_time += (time.time() - self._start_time) + self._start_time = None + + sock = getattr(self, "socket", None) + if sock: + if not isinstance(self.bind_addr, basestring): + # Touch our own socket to make accept() return immediately. + try: + host, port = sock.getsockname()[:2] + except socket.error, x: + if x.args[0] not in socket_errors_to_ignore: + # Changed to use error code and not message + # See http://www.cherrypy.org/ticket/860. + raise + else: + # Note that we're explicitly NOT using AI_PASSIVE, + # here, because we want an actual IP to touch. + # localhost won't work if we've bound to a public IP, + # but it will if we bound to '0.0.0.0' (INADDR_ANY). + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + if hasattr(sock, "close"): + sock.close() + self.socket = None + + self.requests.stop(self.shutdown_timeout) + + +class Gateway(object): + + def __init__(self, req): + self.req = req + + def respond(self): + raise NotImplemented + + +# These may either be wsgiserver.SSLAdapter subclasses or the string names +# of such classes (in which case they will be lazily loaded). +ssl_adapters = { + 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', + 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', + } + +def get_ssl_adapter_class(name='pyopenssl'): + adapter = ssl_adapters[name.lower()] + if isinstance(adapter, basestring): + last_dot = adapter.rfind(".") + attr_name = adapter[last_dot + 1:] + mod_path = adapter[:last_dot] + + try: + mod = sys.modules[mod_path] + if mod is None: + raise KeyError() + except KeyError: + # The last [''] is important. + mod = __import__(mod_path, globals(), locals(), ['']) + + # Let an AttributeError propagate outward. + try: + adapter = getattr(mod, attr_name) + except AttributeError: + raise AttributeError("'%s' object has no attribute '%s'" + % (mod_path, attr_name)) + + return adapter + +# -------------------------------- WSGI Stuff -------------------------------- # + + +class CherryPyWSGIServer(HTTPServer): + + wsgi_version = (1, 0) + + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): + self.requests = ThreadPool(self, min=numthreads or 1, max=max) + self.wsgi_app = wsgi_app + self.gateway = wsgi_gateways[self.wsgi_version] + + self.bind_addr = bind_addr + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.request_queue_size = request_queue_size + + self.timeout = timeout + self.shutdown_timeout = shutdown_timeout + self.clear_stats() + + def _get_numthreads(self): + return self.requests.min + def _set_numthreads(self, value): + self.requests.min = value + numthreads = property(_get_numthreads, _set_numthreads) + + +class WSGIGateway(Gateway): + + def __init__(self, req): + self.req = req + self.started_response = False + self.env = self.get_environ() + self.remaining_bytes_out = None + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + raise NotImplemented + + def respond(self): + response = self.req.server.wsgi_app(self.env, self.start_response) + try: + for chunk in response: + # "The start_response callable must not actually transmit + # the response headers. Instead, it must store them for the + # server or gateway to transmit only after the first + # iteration of the application return value that yields + # a NON-EMPTY string, or upon the application's first + # invocation of the write() callable." (PEP 333) + if chunk: + if isinstance(chunk, unicode): + chunk = chunk.encode('ISO-8859-1') + self.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + + def start_response(self, status, headers, exc_info = None): + """WSGI callable to begin the HTTP response.""" + # "The application may call start_response more than once, + # if and only if the exc_info argument is provided." + if self.started_response and not exc_info: + raise AssertionError("WSGI start_response called a second " + "time with no exc_info.") + self.started_response = True + + # "if exc_info is provided, and the HTTP headers have already been + # sent, start_response must raise an error, and should raise the + # exc_info tuple." + if self.req.sent_headers: + try: + raise exc_info[0], exc_info[1], exc_info[2] + finally: + exc_info = None + + self.req.status = status + for k, v in headers: + if not isinstance(k, str): + raise TypeError("WSGI response header key %r is not a byte string." % k) + if not isinstance(v, str): + raise TypeError("WSGI response header value %r is not a byte string." % v) + if k.lower() == 'content-length': + self.remaining_bytes_out = int(v) + self.req.outheaders.extend(headers) + + return self.write + + def write(self, chunk): + """WSGI callable to write unbuffered data to the client. + + This method is also used internally by start_response (to write + data from the iterable returned by the WSGI application). + """ + if not self.started_response: + raise AssertionError("WSGI write called before start_response.") + + chunklen = len(chunk) + rbo = self.remaining_bytes_out + if rbo is not None and chunklen > rbo: + if not self.req.sent_headers: + # Whew. We can send a 500 to the client. + self.req.simple_response("500 Internal Server Error", + "The requested resource returned more bytes than the " + "declared Content-Length.") + else: + # Dang. We have probably already sent data. Truncate the chunk + # to fit (so the client doesn't hang) and raise an error later. + chunk = chunk[:rbo] + + if not self.req.sent_headers: + self.req.sent_headers = True + self.req.send_headers() + + self.req.write(chunk) + + if rbo is not None: + rbo -= chunklen + if rbo < 0: + raise ValueError( + "Response body exceeds the declared Content-Length.") + + +class WSGIGateway_10(WSGIGateway): + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env = { + # set a non-standard environ entry so the WSGI app can know what + # the *real* server protocol is (and what features to support). + # See http://www.faqs.org/rfcs/rfc2145.html. + 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, + 'PATH_INFO': req.path, + 'QUERY_STRING': req.qs, + 'REMOTE_ADDR': req.conn.remote_addr or '', + 'REMOTE_PORT': str(req.conn.remote_port or ''), + 'REQUEST_METHOD': req.method, + 'REQUEST_URI': req.uri, + 'SCRIPT_NAME': '', + 'SERVER_NAME': req.server.server_name, + # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. + 'SERVER_PROTOCOL': req.request_protocol, + 'SERVER_SOFTWARE': req.server.software, + 'wsgi.errors': sys.stderr, + 'wsgi.input': req.rfile, + 'wsgi.multiprocess': False, + 'wsgi.multithread': True, + 'wsgi.run_once': False, + 'wsgi.url_scheme': req.scheme, + 'wsgi.version': (1, 0), + } + + if isinstance(req.server.bind_addr, basestring): + # AF_UNIX. This isn't really allowed by WSGI, which doesn't + # address unix domain sockets. But it's better than nothing. + env["SERVER_PORT"] = "" + else: + env["SERVER_PORT"] = str(req.server.bind_addr[1]) + + # Request headers + for k, v in req.inheaders.iteritems(): + env["HTTP_" + k.upper().replace("-", "_")] = v + + # CONTENT_TYPE/CONTENT_LENGTH + ct = env.pop("HTTP_CONTENT_TYPE", None) + if ct is not None: + env["CONTENT_TYPE"] = ct + cl = env.pop("HTTP_CONTENT_LENGTH", None) + if cl is not None: + env["CONTENT_LENGTH"] = cl + + if req.conn.ssl_env: + env.update(req.conn.ssl_env) + + return env + + +class WSGIGateway_u0(WSGIGateway_10): + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env_10 = WSGIGateway_10.get_environ(self) + env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) + env[u'wsgi.version'] = ('u', 0) + + # Request-URI + env.setdefault(u'wsgi.url_encoding', u'utf-8') + try: + for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: + env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) + except UnicodeDecodeError: + # Fall back to latin 1 so apps can transcode if needed. + env[u'wsgi.url_encoding'] = u'ISO-8859-1' + for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: + env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) + + for k, v in sorted(env.items()): + if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'): + env[k] = v.decode('ISO-8859-1') + + return env + +wsgi_gateways = { + (1, 0): WSGIGateway_10, + ('u', 0): WSGIGateway_u0, +} + +class WSGIPathInfoDispatcher(object): + """A WSGI dispatcher for dispatch based on the PATH_INFO. + + apps: a dict or list of (path_prefix, app) pairs. + """ + + def __init__(self, apps): + try: + apps = apps.items() + except AttributeError: + pass + + # Sort the apps by len(path), descending + apps.sort(cmp=lambda x,y: cmp(len(x[0]), len(y[0]))) + apps.reverse() + + # The path_prefix strings must start, but not end, with a slash. + # Use "" instead of "/". + self.apps = [(p.rstrip("/"), a) for p, a in apps] + + def __call__(self, environ, start_response): + path = environ["PATH_INFO"] or "/" + for p, app in self.apps: + # The apps list should be sorted by length, descending. + if path.startswith(p + "/") or path == p: + environ = environ.copy() + environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p + environ["PATH_INFO"] = path[len(p):] + return app(environ, start_response) + + start_response('404 Not Found', [('Content-Type', 'text/plain'), + ('Content-Length', '0')]) + return [''] + diff --git a/cherrypy/wsgiserver/ssl_builtin.py b/cherrypy/wsgiserver/ssl_builtin.py new file mode 100755 index 0000000..64c0eeb --- /dev/null +++ b/cherrypy/wsgiserver/ssl_builtin.py @@ -0,0 +1,72 @@ +"""A library for integrating Python's builtin ``ssl`` library with CherryPy. + +The ssl module must be importable for SSL functionality. + +To use this module, set ``CherryPyWSGIServer.ssl_adapter`` to an instance of +``BuiltinSSLAdapter``. +""" + +try: + import ssl +except ImportError: + ssl = None + +from cherrypy import wsgiserver + + +class BuiltinSSLAdapter(wsgiserver.SSLAdapter): + """A wrapper for integrating Python's builtin ssl module with CherryPy.""" + + certificate = None + """The filename of the server SSL certificate.""" + + private_key = None + """The filename of the server's private key file.""" + + def __init__(self, certificate, private_key, certificate_chain=None): + if ssl is None: + raise ImportError("You must install the ssl module to use HTTPS.") + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + + def bind(self, sock): + """Wrap and return the given socket.""" + return sock + + def wrap(self, sock): + """Wrap and return the given socket, plus WSGI environ entries.""" + try: + s = ssl.wrap_socket(sock, do_handshake_on_connect=True, + server_side=True, certfile=self.certificate, + keyfile=self.private_key, ssl_version=ssl.PROTOCOL_SSLv23) + except ssl.SSLError, e: + if e.errno == ssl.SSL_ERROR_EOF: + # This is almost certainly due to the cherrypy engine + # 'pinging' the socket to assert it's connectable; + # the 'ping' isn't SSL. + return None, {} + elif e.errno == ssl.SSL_ERROR_SSL: + if e.args[1].endswith('http request'): + # The client is speaking HTTP to an HTTPS server. + raise wsgiserver.NoSSLError + raise + return s, self.get_environ(s) + + # TODO: fill this out more with mod ssl env + def get_environ(self, sock): + """Create WSGI environ entries to be merged into each request.""" + cipher = sock.cipher() + ssl_environ = { + "wsgi.url_scheme": "https", + "HTTPS": "on", + 'SSL_PROTOCOL': cipher[1], + 'SSL_CIPHER': cipher[0] +## SSL_VERSION_INTERFACE string The mod_ssl program version +## SSL_VERSION_LIBRARY string The OpenSSL program version + } + return ssl_environ + + def makefile(self, sock, mode='r', bufsize=-1): + return wsgiserver.CP_fileobject(sock, mode, bufsize) + diff --git a/cherrypy/wsgiserver/ssl_pyopenssl.py b/cherrypy/wsgiserver/ssl_pyopenssl.py new file mode 100755 index 0000000..f3d9bf5 --- /dev/null +++ b/cherrypy/wsgiserver/ssl_pyopenssl.py @@ -0,0 +1,256 @@ +"""A library for integrating pyOpenSSL with CherryPy. + +The OpenSSL module must be importable for SSL functionality. +You can obtain it from http://pyopenssl.sourceforge.net/ + +To use this module, set CherryPyWSGIServer.ssl_adapter to an instance of +SSLAdapter. There are two ways to use SSL: + +Method One +---------- + + * ``ssl_adapter.context``: an instance of SSL.Context. + +If this is not None, it is assumed to be an SSL.Context instance, +and will be passed to SSL.Connection on bind(). The developer is +responsible for forming a valid Context object. This approach is +to be preferred for more flexibility, e.g. if the cert and key are +streams instead of files, or need decryption, or SSL.SSLv3_METHOD +is desired instead of the default SSL.SSLv23_METHOD, etc. Consult +the pyOpenSSL documentation for complete options. + +Method Two (shortcut) +--------------------- + + * ``ssl_adapter.certificate``: the filename of the server SSL certificate. + * ``ssl_adapter.private_key``: the filename of the server's private key file. + +Both are None by default. If ssl_adapter.context is None, but .private_key +and .certificate are both given and valid, they will be read, and the +context will be automatically created from them. +""" + +import socket +import threading +import time + +from cherrypy import wsgiserver + +try: + from OpenSSL import SSL + from OpenSSL import crypto +except ImportError: + SSL = None + + +class SSL_fileobject(wsgiserver.CP_fileobject): + """SSL file object attached to a socket object.""" + + ssl_timeout = 3 + ssl_retry = .01 + + def _safe_call(self, is_reader, call, *args, **kwargs): + """Wrap the given call with SSL error-trapping. + + is_reader: if False EOF errors will be raised. If True, EOF errors + will return "" (to emulate normal sockets). + """ + start = time.time() + while True: + try: + return call(*args, **kwargs) + except SSL.WantReadError: + # Sleep and try again. This is dangerous, because it means + # the rest of the stack has no way of differentiating + # between a "new handshake" error and "client dropped". + # Note this isn't an endless loop: there's a timeout below. + time.sleep(self.ssl_retry) + except SSL.WantWriteError: + time.sleep(self.ssl_retry) + except SSL.SysCallError, e: + if is_reader and e.args == (-1, 'Unexpected EOF'): + return "" + + errnum = e.args[0] + if is_reader and errnum in wsgiserver.socket_errors_to_ignore: + return "" + raise socket.error(errnum) + except SSL.Error, e: + if is_reader and e.args == (-1, 'Unexpected EOF'): + return "" + + thirdarg = None + try: + thirdarg = e.args[0][0][2] + except IndexError: + pass + + if thirdarg == 'http request': + # The client is talking HTTP to an HTTPS server. + raise wsgiserver.NoSSLError() + + raise wsgiserver.FatalSSLAlert(*e.args) + except: + raise + + if time.time() - start > self.ssl_timeout: + raise socket.timeout("timed out") + + def recv(self, *args, **kwargs): + buf = [] + r = super(SSL_fileobject, self).recv + while True: + data = self._safe_call(True, r, *args, **kwargs) + buf.append(data) + p = self._sock.pending() + if not p: + return "".join(buf) + + def sendall(self, *args, **kwargs): + return self._safe_call(False, super(SSL_fileobject, self).sendall, + *args, **kwargs) + + def send(self, *args, **kwargs): + return self._safe_call(False, super(SSL_fileobject, self).send, + *args, **kwargs) + + +class SSLConnection: + """A thread-safe wrapper for an SSL.Connection. + + ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``. + """ + + def __init__(self, *args): + self._ssl_conn = SSL.Connection(*args) + self._lock = threading.RLock() + + for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', + 'renegotiate', 'bind', 'listen', 'connect', 'accept', + 'setblocking', 'fileno', 'close', 'get_cipher_list', + 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', + 'makefile', 'get_app_data', 'set_app_data', 'state_string', + 'sock_shutdown', 'get_peer_certificate', 'want_read', + 'want_write', 'set_connect_state', 'set_accept_state', + 'connect_ex', 'sendall', 'settimeout', 'gettimeout'): + exec("""def %s(self, *args): + self._lock.acquire() + try: + return self._ssl_conn.%s(*args) + finally: + self._lock.release() +""" % (f, f)) + + def shutdown(self, *args): + self._lock.acquire() + try: + # pyOpenSSL.socket.shutdown takes no args + return self._ssl_conn.shutdown() + finally: + self._lock.release() + + +class pyOpenSSLAdapter(wsgiserver.SSLAdapter): + """A wrapper for integrating pyOpenSSL with CherryPy.""" + + context = None + """An instance of SSL.Context.""" + + certificate = None + """The filename of the server SSL certificate.""" + + private_key = None + """The filename of the server's private key file.""" + + certificate_chain = None + """Optional. The filename of CA's intermediate certificate bundle. + + This is needed for cheaper "chained root" SSL certificates, and should be + left as None if not required.""" + + def __init__(self, certificate, private_key, certificate_chain=None): + if SSL is None: + raise ImportError("You must install pyOpenSSL to use HTTPS.") + + self.context = None + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + self._environ = None + + def bind(self, sock): + """Wrap and return the given socket.""" + if self.context is None: + self.context = self.get_context() + conn = SSLConnection(self.context, sock) + self._environ = self.get_environ() + return conn + + def wrap(self, sock): + """Wrap and return the given socket, plus WSGI environ entries.""" + return sock, self._environ.copy() + + def get_context(self): + """Return an SSL.Context from self attributes.""" + # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 + c = SSL.Context(SSL.SSLv23_METHOD) + c.use_privatekey_file(self.private_key) + if self.certificate_chain: + c.load_verify_locations(self.certificate_chain) + c.use_certificate_file(self.certificate) + return c + + def get_environ(self): + """Return WSGI environ entries to be merged into each request.""" + ssl_environ = { + "HTTPS": "on", + # pyOpenSSL doesn't provide access to any of these AFAICT +## 'SSL_PROTOCOL': 'SSLv2', +## SSL_CIPHER string The cipher specification name +## SSL_VERSION_INTERFACE string The mod_ssl program version +## SSL_VERSION_LIBRARY string The OpenSSL program version + } + + if self.certificate: + # Server certificate attributes + cert = open(self.certificate, 'rb').read() + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) + ssl_environ.update({ + 'SSL_SERVER_M_VERSION': cert.get_version(), + 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), +## 'SSL_SERVER_V_START': Validity of server's certificate (start time), +## 'SSL_SERVER_V_END': Validity of server's certificate (end time), + }) + + for prefix, dn in [("I", cert.get_issuer()), + ("S", cert.get_subject())]: + # X509Name objects don't seem to have a way to get the + # complete DN string. Use str() and slice it instead, + # because str(dn) == "<X509Name object '/C=US/ST=...'>" + dnstr = str(dn)[18:-2] + + wsgikey = 'SSL_SERVER_%s_DN' % prefix + ssl_environ[wsgikey] = dnstr + + # The DN should be of the form: /k1=v1/k2=v2, but we must allow + # for any value to contain slashes itself (in a URL). + while dnstr: + pos = dnstr.rfind("=") + dnstr, value = dnstr[:pos], dnstr[pos + 1:] + pos = dnstr.rfind("/") + dnstr, key = dnstr[:pos], dnstr[pos + 1:] + if key and value: + wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) + ssl_environ[wsgikey] = value + + return ssl_environ + + def makefile(self, sock, mode='r', bufsize=-1): + if SSL and isinstance(sock, SSL.ConnectionType): + timeout = sock.gettimeout() + f = SSL_fileobject(sock, mode, bufsize) + f.ssl_timeout = timeout + return f + else: + return wsgiserver.CP_fileobject(sock, mode, bufsize) + |
