web2py.gluon.rewrite

Package web2py :: Package gluon :: Module rewrite |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.rewrite

       1  #!/bin/env python
       2  # -*- coding: utf-8 -*-
       3
       4  """
       5  This file is part of the web2py Web Framework
       6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
       7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
       8
       9  gluon.rewrite parses incoming URLs and formats outgoing URLs for gluon.html.URL.
      10
      11  In addition, it rewrites both incoming and outgoing URLs based on the (optional) user-supplied routes.py,
      12  which also allows for rewriting of certain error messages.
      13
      14  routes.py supports two styles of URL rewriting, depending on whether 'routers' is defined.
      15  Refer to router.example.py and routes.example.py for additional documentation.
      16
      17  """
      18
      19  import os
      20  import re
      21  import logging
      22  import traceback
      23  import threading
      24  import urllib
      25  from storage import Storage, List
      26  from http import HTTP
      27  from fileutils import abspath, read_file
      28  from settings import global_settings
      29
      30  logger = logging.getLogger('web2py.rewrite')
      31
      32  thread = threading.local()  # thread-local storage for routing parameters
      33
      34 -def _router_default():
      35      "return new copy of default base router"
      36      router = Storage(
      37          default_application = 'init',
      38              applications = 'ALL',
      39          default_controller = 'default',
      40              controllers = 'DEFAULT',
      41          default_function = 'index',
      42              functions = dict(),
      43          default_language = None,
      44              languages = None,
      45          root_static = ['favicon.ico', 'robots.txt'],
      46          domains = None,
      47          exclusive_domain = False,
      48          map_hyphen = False,
      49          acfe_match = r'\w+$',              # legal app/ctlr/fcn/ext
      50          file_match = r'(\w+[-=./]?)+$',    # legal file (path) name
      51          args_match = r'([\w@ -]+[=.]?)*$', # legal arg in args
      52      )
      53      return router
      54
      55 -def _params_default(app=None):
      56      "return new copy of default parameters"
      57      p = Storage()
      58      p.name = app or "BASE"
      59      p.default_application = app or "init"
      60      p.default_controller = "default"
      61      p.default_function = "index"
      62      p.routes_app = []
      63      p.routes_in = []
      64      p.routes_out = []
      65      p.routes_onerror = []
      66      p.routes_apps_raw = []
      67      p.error_handler = None
      68      p.error_message = '<html><body><h1>%s</h1></body></html>'
      69      p.error_message_ticket = \
      70          '<html><body><h1>Internal error</h1>Ticket issued: <a href="/admin/default/ticket/%(ticket)s" target="_blank">%(ticket)s</a></body><!-- this is junk text else IE does not display the page: '+('x'*512)+' //--></html>'
      71      p.routers = None
      72      p.logging = 'off'
      73      return p
      74
      75  params_apps = dict()
      76  params = _params_default(app=None)  # regex rewrite parameters
      77  thread.routes = params              # default to base regex rewrite parameters
      78  routers = None
      79
      80 -def log_rewrite(string):
      81      "Log rewrite activity under control of routes.py"
      82      if params.logging == 'debug':   # catch common cases first
      83          logger.debug(string)
      84      elif params.logging == 'off' or not params.logging:
      85          pass
      86      elif params.logging == 'print':
      87          print string
      88      elif params.logging == 'info':
      89          logger.info(string)
      90      elif params.logging == 'warning':
      91          logger.warning(string)
      92      elif params.logging == 'error':
      93          logger.error(string)
      94      elif params.logging == 'critical':
      95          logger.critical(string)
      96      else:
      97          logger.debug(string)
      98
      99  ROUTER_KEYS = set(('default_application', 'applications', 'default_controller', 'controllers',
     100      'default_function', 'functions', 'default_language', 'languages',
     101      'domain', 'domains', 'root_static', 'path_prefix',
     102      'exclusive_domain', 'map_hyphen', 'map_static',
     103      'acfe_match', 'file_match', 'args_match'))
     104
     105  ROUTER_BASE_KEYS = set(('applications', 'default_application', 'domains', 'path_prefix'))
     106
     107  #  The external interface to rewrite consists of:
     108  #
     109  #  load: load routing configuration file(s)
     110  #  url_in: parse and rewrite incoming URL
     111  #  url_out: assemble and rewrite outgoing URL
     112  #
     113  #  thread.routes.default_application
     114  #  thread.routes.error_message
     115  #  thread.routes.error_message_ticket
     116  #  thread.routes.try_redirect_on_error
     117  #  thread.routes.error_handler
     118  #
     119  #  filter_url: helper for doctest & unittest
     120  #  filter_err: helper for doctest & unittest
     121  #  regex_filter_out: doctest
     122
     123 -def url_in(request, environ):
     124      "parse and rewrite incoming URL"
     125      if routers:
     126          return map_url_in(request, environ)
     127      return regex_url_in(request, environ)
     128
     129 -def url_out(request, env, application, controller, function, args, other, scheme, host, port):
     130      "assemble and rewrite outgoing URL"
     131      if routers:
     132          acf = map_url_out(request, env, application, controller, function, args, other, scheme, host, port)
     133          url = '%s%s' % (acf, other)
     134      else:
     135          url = '/%s/%s/%s%s' % (application, controller, function, other)
     136          url = regex_filter_out(url, env)
     137      #
     138      #  fill in scheme and host if absolute URL is requested
     139      #  scheme can be a string, eg 'http', 'https', 'ws', 'wss'
     140      #
     141      if scheme or port is not None:
     142          if host is None:    # scheme or port implies host
     143              host = True
     144      if not scheme or scheme is True:
     145          if request and request.env:
     146              scheme = request.env.get('wsgi_url_scheme', 'http').lower()
     147          else:
     148              scheme = 'http' # some reasonable default in case we need it
     149      if host is not None:
     150          if host is True:
     151              host = request.env.http_host
     152      if host:
     153          if port is None:
     154              port = ''
     155          else:
     156              port = ':%s' % port
     157          url = '%s://%s%s%s' % (scheme, host, port, url)
     158      return url
     159
     160 -def try_rewrite_on_error(http_response, request, environ, ticket=None):
     161      """
     162      called from main.wsgibase to rewrite the http response.
     163      """
     164      status = int(str(http_response.status).split()[0])
     165      if status>=399 and thread.routes.routes_onerror:
     166          keys=set(('%s/%s' % (request.application, status),
     167                    '%s/*' % (request.application),
     168                    '*/%s' % (status),
     169                    '*/*'))
     170          for (key,uri) in thread.routes.routes_onerror:
     171              if key in keys:
     172                  if uri == '!':
     173                      # do nothing!
     174                      return http_response, environ
     175                  elif '?' in uri:
     176                      path_info, query_string = uri.split('?',1)
     177                      query_string += '&'
     178                  else:
     179                      path_info, query_string = uri, ''
     180                  query_string += \
     181                      'code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
     182                      (status,ticket,request.env.request_uri,request.url)
     183                  if uri.startswith('http://') or uri.startswith('https://'):
     184                      # make up a response
     185                      url = path_info+'?'+query_string
     186                      message = 'You are being redirected <a href="%s">here</a>'
     187                      return HTTP(303, message % url, Location=url), environ
     188                  else:
     189                      error_raising_path = environ['PATH_INFO']
     190                      # Rewrite routes_onerror path.
     191                      path_info = '/' + path_info.lstrip('/') # add leading '/' if missing
     192                      environ['PATH_INFO'] = path_info
     193                      error_handling_path = url_in(request, environ)[1]['PATH_INFO']
     194                      # Avoid infinite loop.
     195                      if error_handling_path != error_raising_path:
     196                          # wsgibase will be called recursively with the routes_onerror path.
     197                          environ['PATH_INFO'] = path_info
     198                          environ['QUERY_STRING'] = query_string
     199                          return None, environ
     200      # do nothing!
     201      return http_response, environ
     202
     203 -def try_redirect_on_error(http_object, request, ticket=None):
     204      "called from main.wsgibase to rewrite the http response"
     205      status = int(str(http_object.status).split()[0])
     206      if status>399 and thread.routes.routes_onerror:
     207          keys=set(('%s/%s' % (request.application, status),
     208                    '%s/*' % (request.application),
     209                    '*/%s' % (status),
     210                    '*/*'))
     211          for (key,redir) in thread.routes.routes_onerror:
     212              if key in keys:
     213                  if redir == '!':
     214                      break
     215                  elif '?' in redir:
     216                      url = '%s&code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
     217                          (redir,status,ticket,request.env.request_uri,request.url)
     218                  else:
     219                      url = '%s?code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
     220                          (redir,status,ticket,request.env.request_uri,request.url)
     221                  return HTTP(303,
     222                              'You are being redirected <a href="%s">here</a>' % url,
     223                              Location=url)
     224      return http_object
     225
     226
     227 -def load(routes='routes.py', app=None, data=None, rdict=None):
     228      """
     229      load: read (if file) and parse routes
     230      store results in params
     231      (called from main.py at web2py initialization time)
     232      If data is present, it's used instead of the routes.py contents.
     233      If rdict is present, it must be a dict to be used for routers (unit test)
     234      """
     235      global params
     236      global routers
     237      if app is None:
     238          # reinitialize
     239          global params_apps
     240          params_apps = dict()
     241          params = _params_default(app=None)  # regex rewrite parameters
     242          thread.routes = params              # default to base regex rewrite parameters
     243          routers = None
     244
     245      if isinstance(rdict, dict):
     246          symbols = dict(routers=rdict)
     247          path = 'rdict'
     248      else:
     249          if data is not None:
     250              path = 'routes'
     251          else:
     252              if app is None:
     253                  path = abspath(routes)
     254              else:
     255                  path = abspath('applications', app, routes)
     256              if not os.path.exists(path):
     257                  return
     258              data = read_file(path).replace('\r\n','\n')
     259
     260          symbols = {}
     261          try:
     262              exec (data + '\n') in symbols
     263          except SyntaxError, e:
     264              logger.error(
     265                  '%s has a syntax error and will not be loaded\n' % path
     266                  + traceback.format_exc())
     267              raise e
     268
     269      p = _params_default(app)
     270
     271      for sym in ('routes_app', 'routes_in', 'routes_out'):
     272          if sym in symbols:
     273              for (k, v) in symbols[sym]:
     274                  p[sym].append(compile_regex(k, v))
     275      for sym in ('routes_onerror', 'routes_apps_raw',
     276                  'error_handler','error_message', 'error_message_ticket',
     277                  'default_application','default_controller', 'default_function',
     278                  'logging'):
     279          if sym in symbols:
     280              p[sym] = symbols[sym]
     281      if 'routers' in symbols:
     282          p.routers = Storage(symbols['routers'])
     283          for key in p.routers:
     284              if isinstance(p.routers[key], dict):
     285                  p.routers[key] = Storage(p.routers[key])
     286
     287      if app is None:
     288          params = p                  # install base rewrite parameters
     289          thread.routes = params      # install default as current routes
     290          #
     291          #  create the BASE router if routers in use
     292          #
     293          routers = params.routers    # establish routers if present
     294          if isinstance(routers, dict):
     295              routers = Storage(routers)
     296          if routers is not None:
     297              router = _router_default()
     298              if routers.BASE:
     299                  router.update(routers.BASE)
     300              routers.BASE = router
     301
     302          #  scan each app in applications/
     303          #    create a router, if routers are in use
     304          #    parse the app-specific routes.py if present
     305          #
     306          all_apps = []
     307          for appname in [app for app in os.listdir(abspath('applications')) if not app.startswith('.')]:
     308              if os.path.isdir(abspath('applications', appname)) and \
     309                 os.path.isdir(abspath('applications', appname, 'controllers')):
     310                  all_apps.append(appname)
     311                  if routers:
     312                      router = Storage(routers.BASE)   # new copy
     313                      if appname in routers:
     314                          for key in routers[appname].keys():
     315                              if key in ROUTER_BASE_KEYS:
     316                                  raise SyntaxError, "BASE-only key '%s' in router '%s'" % (key, appname)
     317                          router.update(routers[appname])
     318                      routers[appname] = router
     319                  if os.path.exists(abspath('applications', appname, routes)):
     320                      load(routes, appname)
     321
     322          if routers:
     323              load_routers(all_apps)
     324
     325      else: # app
     326          params_apps[app] = p
     327          if routers and p.routers:
     328              if app in p.routers:
     329                  routers[app].update(p.routers[app])
     330
     331      log_rewrite('URL rewrite is on. configuration in %s' % path)
     332
     333
     334  regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
     335  regex_anything = re.compile(r'(?<!\\)\$anything')
     336
     337 -def compile_regex(k, v):
     338      """
     339      Preprocess and compile the regular expressions in routes_app/in/out
     340
     341      The resulting regex will match a pattern of the form:
     342
     343          [remote address]:[protocol]://[host]:[method] [path]
     344
     345      We allow abbreviated regexes on input; here we try to complete them.
     346      """
     347      k0 = k  # original k for error reporting
     348      # bracket regex in ^...$ if not already done
     349      if not k[0] == '^':
     350          k = '^%s' % k
     351      if not k[-1] == '$':
     352          k = '%s$' % k
     353      # if there are no :-separated parts, prepend a catch-all for the IP address
     354      if k.find(':') < 0:
     355          # k = '^.*?:%s' % k[1:]
     356          k = '^.*?:https?://[^:/]+:[a-z]+ %s' % k[1:]
     357      # if there's no ://, provide a catch-all for the protocol, host & method
     358      if k.find('://') < 0:
     359          i = k.find(':/')
     360          if i < 0:
     361              raise SyntaxError, "routes pattern syntax error: path needs leading '/' [%s]" % k0
     362          k = r'%s:https?://[^:/]+:[a-z]+ %s' % (k[:i], k[i+1:])
     363      # $anything -> ?P<anything>.*
     364      for item in regex_anything.findall(k):
     365          k = k.replace(item, '(?P<anything>.*)')
     366      # $a (etc) -> ?P<a>\w+
     367      for item in regex_at.findall(k):
     368          k = k.replace(item, r'(?P<%s>\w+)' % item[1:])
     369      # same for replacement pattern, but with \g
     370      for item in regex_at.findall(v):
     371          v = v.replace(item, r'\g<%s>' % item[1:])
     372      return (re.compile(k, re.DOTALL), v)
     373
     374 -def load_routers(all_apps):
     375      "load-time post-processing of routers"
     376
     377      for app in routers.keys():
     378          # initialize apps with routers that aren't present, on behalf of unit tests
     379          if app not in all_apps:
     380              all_apps.append(app)
     381              router = Storage(routers.BASE)   # new copy
     382              if app != 'BASE':
     383                  for key in routers[app].keys():
     384                      if key in ROUTER_BASE_KEYS:
     385                          raise SyntaxError, "BASE-only key '%s' in router '%s'" % (key, app)
     386              router.update(routers[app])
     387              routers[app] = router
     388          router = routers[app]
     389          for key in router.keys():
     390              if key not in ROUTER_KEYS:
     391                  raise SyntaxError, "unknown key '%s' in router '%s'" % (key, app)
     392          if not router.controllers:
     393              router.controllers = set()
     394          elif not isinstance(router.controllers, str):
     395              router.controllers = set(router.controllers)
     396          if router.languages:
     397              router.languages = set(router.languages)
     398          else:
     399              router.languages = set()
     400          if app != 'BASE':
     401              for base_only in ROUTER_BASE_KEYS:
     402                  router.pop(base_only, None)
     403              if 'domain' in router:
     404                  routers.BASE.domains[router.domain] = app
     405              if isinstance(router.controllers, str) and router.controllers == 'DEFAULT':
     406                  router.controllers = set()
     407                  if os.path.isdir(abspath('applications', app)):
     408                      cpath = abspath('applications', app, 'controllers')
     409                      for cname in os.listdir(cpath):
     410                          if os.path.isfile(abspath(cpath, cname)) and cname.endswith('.py'):
     411                              router.controllers.add(cname[:-3])
     412              if router.controllers:
     413                  router.controllers.add('static')
     414                  router.controllers.add(router.default_controller)
     415              if router.functions:
     416                  if isinstance(router.functions, (set, tuple, list)):
     417                      functions = set(router.functions)
     418                      if isinstance(router.default_function, str):
     419                          functions.add(router.default_function)  # legacy compatibility
     420                      router.functions = { router.default_controller: functions }
     421                  for controller in router.functions:
     422                      router.functions[controller] = set(router.functions[controller])
     423              else:
     424                  router.functions = dict()
     425
     426      if isinstance(routers.BASE.applications, str) and routers.BASE.applications == 'ALL':
     427          routers.BASE.applications = list(all_apps)
     428      if routers.BASE.applications:
     429          routers.BASE.applications = set(routers.BASE.applications)
     430      else:
     431          routers.BASE.applications = set()
     432
     433      for app in routers.keys():
     434          # set router name
     435          router = routers[app]
     436          router.name = app
     437          # compile URL validation patterns
     438          router._acfe_match = re.compile(router.acfe_match)
     439          router._file_match = re.compile(router.file_match)
     440          if router.args_match:
     441              router._args_match = re.compile(router.args_match)
     442          # convert path_prefix to a list of path elements
     443          if router.path_prefix:
     444              if isinstance(router.path_prefix, str):
     445                  router.path_prefix = router.path_prefix.strip('/').split('/')
     446
     447      #  rewrite BASE.domains as tuples
     448      #
     449      #      key:   'domain[:port]' -> (domain, port)
     450      #      value: 'application[/controller] -> (application, controller)
     451      #      (port and controller may be None)
     452      #
     453      domains = dict()
     454      if routers.BASE.domains:
     455          for (domain, app) in [(d.strip(':'), a.strip('/')) for (d, a) in routers.BASE.domains.items()]:
     456              port = None
     457              if ':' in domain:
     458                  (domain, port) = domain.split(':')
     459              ctlr = None
     460              fcn = None
     461              if '/' in app:
     462                  (app, ctlr) = app.split('/', 1)
     463              if ctlr and '/' in ctlr:
     464                  (ctlr, fcn) = ctlr.split('/')
     465              if app not in all_apps and app not in routers:
     466                  raise SyntaxError, "unknown app '%s' in domains" % app
     467              domains[(domain, port)] = (app, ctlr, fcn)
     468      routers.BASE.domains = domains
     469
     470 -def regex_uri(e, regexes, tag, default=None):
     471      "filter incoming URI against a list of regexes"
     472      path = e['PATH_INFO']
     473      host = e.get('HTTP_HOST', 'localhost').lower()
     474      i = host.find(':')
     475      if i > 0:
     476          host = host[:i]
     477      key = '%s:%s://%s:%s %s' % \
     478          (e.get('REMOTE_ADDR','localhost'),
     479           e.get('wsgi.url_scheme', 'http').lower(), host,
     480           e.get('REQUEST_METHOD', 'get').lower(), path)
     481      for (regex, value) in regexes:
     482          if regex.match(key):
     483              rewritten = regex.sub(value, key)
     484              log_rewrite('%s: [%s] [%s] -> %s' % (tag, key, value, rewritten))
     485              return rewritten
     486      log_rewrite('%s: [%s] -> %s (not rewritten)' % (tag, key, default))
     487      return default
     488
     489 -def regex_select(env=None, app=None, request=None):
     490      """
     491      select a set of regex rewrite params for the current request
     492      """
     493      if app:
     494          thread.routes = params_apps.get(app, params)
     495      elif env and params.routes_app:
     496          if routers:
     497              map_url_in(request, env, app=True)
     498          else:
     499              app = regex_uri(env, params.routes_app, "routes_app")
     500              thread.routes = params_apps.get(app, params)
     501      else:
     502          thread.routes = params # default to base rewrite parameters
     503      log_rewrite("select routing parameters: %s" % thread.routes.name)
     504      return app  # for doctest
     505
     506 -def regex_filter_in(e):
     507      "regex rewrite incoming URL"
     508      query = e.get('QUERY_STRING', None)
     509      e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
     510      if thread.routes.routes_in:
     511          path = regex_uri(e, thread.routes.routes_in, "routes_in", e['PATH_INFO'])
     512          items = path.split('?', 1)
     513          e['PATH_INFO'] = items[0]
     514          if len(items) > 1:
     515              if query:
     516                  query = items[1] + '&' + query
     517              else:
     518                  query = items[1]
     519              e['QUERY_STRING'] = query
     520      e['REQUEST_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
     521      return e
     522
     523
     524  # pattern to replace spaces with underscore in URL
     525  #   also the html escaped variants '+' and '%20' are covered
     526  regex_space = re.compile('(\+|\s|%20)+')
     527
     528  # pattern to find valid paths in url /application/controller/...
     529  #   this could be:
     530  #     for static pages:
     531  #        /<b:application>/static/<x:file>
     532  #     for dynamic pages:
     533  #        /<a:application>[/<c:controller>[/<f:function>[.<e:ext>][/<s:args>]]]
     534  #   application, controller, function and ext may only contain [a-zA-Z0-9_]
     535  #   file and args may also contain '-', '=', '.' and '/'
     536  #   apps in routes_apps_raw must parse raw_args into args
     537
     538  regex_static = re.compile(r'''
     539       (^                              # static pages
     540           /(?P<b> \w+)                # b=app
     541           /static                     # /b/static
     542           /(?P<x> (\w[\-\=\./]?)* )   # x=file
     543       $)
     544       ''', re.X)
     545
     546  regex_url = re.compile(r'''
     547       (^(                                  # (/a/c/f.e/s)
     548           /(?P<a> [\w\s+]+ )               # /a=app
     549           (                                # (/c.f.e/s)
     550               /(?P<c> [\w\s+]+ )           # /a/c=controller
     551               (                            # (/f.e/s)
     552                   /(?P<f> [\w\s+]+ )       # /a/c/f=function
     553                   (                        # (.e)
     554                       \.(?P<e> [\w\s+]+ )  # /a/c/f.e=extension
     555                   )?
     556                   (                        # (/s)
     557                       /(?P<r>              # /a/c/f.e/r=raw_args
     558                       .*
     559                       )
     560                   )?
     561               )?
     562           )?
     563       )?
     564       /?$)
     565       ''', re.X)
     566
     567  regex_args = re.compile(r'''
     568       (^
     569           (?P<s>
     570               ( [\w@/-][=.]? )*          # s=args
     571           )?
     572       /?$)    # trailing slash
     573       ''', re.X)
     574
     575 -def regex_url_in(request, environ):
     576      "rewrite and parse incoming URL"
     577
     578      # ##################################################
     579      # select application
     580      # rewrite URL if routes_in is defined
     581      # update request.env
     582      # ##################################################
     583
     584      regex_select(env=environ, request=request)
     585
     586      if thread.routes.routes_in:
     587          environ = regex_filter_in(environ)
     588
     589      for (key, value) in environ.items():
     590          request.env[key.lower().replace('.', '_')] = value
     591
     592      path = request.env.path_info.replace('\\', '/')
     593
     594      # ##################################################
     595      # serve if a static file
     596      # ##################################################
     597
     598      match = regex_static.match(regex_space.sub('_', path))
     599      if match and match.group('x'):
     600          static_file = os.path.join(request.env.applications_parent,
     601                                     'applications', match.group('b'),
     602                                     'static', match.group('x'))
     603          return (static_file, environ)
     604
     605      # ##################################################
     606      # parse application, controller and function
     607      # ##################################################
     608
     609      path = re.sub('%20', ' ', path)
     610      match = regex_url.match(path)
     611      if not match or match.group('c') == 'static':
     612          raise HTTP(400,
     613                     thread.routes.error_message % 'invalid request',
     614                     web2py_error='invalid path')
     615
     616      request.application = \
     617          regex_space.sub('_', match.group('a') or thread.routes.default_application)
     618      request.controller = \
     619          regex_space.sub('_', match.group('c') or thread.routes.default_controller)
     620      request.function = \
     621          regex_space.sub('_', match.group('f') or thread.routes.default_function)
     622      group_e = match.group('e')
     623      request.raw_extension = group_e and regex_space.sub('_', group_e) or None
     624      request.extension = request.raw_extension or 'html'
     625      request.raw_args = match.group('r')
     626      request.args = List([])
     627      if request.application in thread.routes.routes_apps_raw:
     628          # application is responsible for parsing args
     629          request.args = None
     630      elif request.raw_args:
     631          match = regex_args.match(request.raw_args.replace(' ', '_'))
     632          if match:
     633              group_s = match.group('s')
     634              request.args = \
     635                  List((group_s and group_s.split('/')) or [])
     636              if request.args and request.args[-1] == '':
     637                  request.args.pop()  # adjust for trailing empty arg
     638          else:
     639              raise HTTP(400,
     640                         thread.routes.error_message % 'invalid request',
     641                         web2py_error='invalid path (args)')
     642      return (None, environ)
     643
     644
     645 -def regex_filter_out(url, e=None):
     646      "regex rewrite outgoing URL"
     647      if not hasattr(thread, 'routes'):
     648          regex_select()    # ensure thread.routes is set (for application threads)
     649      if routers:
     650          return url  # already filtered
     651      if thread.routes.routes_out:
     652          items = url.split('?', 1)
     653          if e:
     654              host = e.get('http_host', 'localhost').lower()
     655              i = host.find(':')
     656              if i > 0:
     657                  host = host[:i]
     658              items[0] = '%s:%s://%s:%s %s' % \
     659                   (e.get('remote_addr', ''),
     660                    e.get('wsgi_url_scheme', 'http').lower(), host,
     661                    e.get('request_method', 'get').lower(), items[0])
     662          else:
     663              items[0] = ':http://localhost:get %s' % items[0]
     664          for (regex, value) in thread.routes.routes_out:
     665              if regex.match(items[0]):
     666                  rewritten = '?'.join([regex.sub(value, items[0])] + items[1:])
     667                  log_rewrite('routes_out: [%s] -> %s' % (url, rewritten))
     668                  return rewritten
     669      log_rewrite('routes_out: [%s] not rewritten' % url)
     670      return url
     671
     672
     673 -def filter_url(url, method='get', remote='0.0.0.0', out=False, app=False, lang=None,
     674          domain=(None,None), env=False, scheme=None, host=None, port=None):
     675      "doctest/unittest interface to regex_filter_in() and regex_filter_out()"
     676      regex_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')
     677      match = regex_url.match(url)
     678      urlscheme = match.group('scheme').lower()
     679      urlhost = match.group('host').lower()
     680      uri = match.group('uri')
     681      k = uri.find('?')
     682      if k < 0:
     683          k = len(uri)
     684      if isinstance(domain, str):
     685          domain = (domain, None)
     686      (path_info, query_string) = (uri[:k], uri[k+1:])
     687      path_info = urllib.unquote(path_info)   # simulate server
     688      e = {
     689           'REMOTE_ADDR': remote,
     690           'REQUEST_METHOD': method,
     691           'wsgi.url_scheme': urlscheme,
     692           'HTTP_HOST': urlhost,
     693           'REQUEST_URI': uri,
     694           'PATH_INFO': path_info,
     695           'QUERY_STRING': query_string,
     696           #for filter_out request.env use lowercase
     697           'remote_addr': remote,
     698           'request_method': method,
     699           'wsgi_url_scheme': urlscheme,
     700           'http_host': urlhost
     701      }
     702
     703      request = Storage()
     704      e["applications_parent"] = global_settings.applications_parent
     705      request.env = Storage(e)
     706      request.uri_language = lang
     707
     708      #  determine application only
     709      #
     710      if app:
     711          if routers:
     712              return map_url_in(request, e, app=True)
     713          return regex_select(e)
     714
     715      #  rewrite outbound URL
     716      #
     717      if out:
     718          (request.env.domain_application, request.env.domain_controller) = domain
     719          items = path_info.lstrip('/').split('/')
     720          if items[-1] == '':
     721              items.pop() # adjust trailing empty args
     722          assert len(items) >= 3, "at least /a/c/f is required"
     723          a = items.pop(0)
     724          c = items.pop(0)
     725          f = items.pop(0)
     726          if not routers:
     727              return regex_filter_out(uri, e)
     728          acf = map_url_out(request, None, a, c, f, items, None, scheme, host, port)
     729          if items:
     730              url = '%s/%s' % (acf, '/'.join(items))
     731              if items[-1] == '':
     732                  url += '/'
     733          else:
     734              url = acf
     735          if query_string:
     736              url += '?' + query_string
     737          return url
     738
     739      #  rewrite inbound URL
     740      #
     741      (static, e) = url_in(request, e)
     742      if static:
     743          return static
     744      result = "/%s/%s/%s" % (request.application, request.controller, request.function)
     745      if request.extension and request.extension != 'html':
     746          result += ".%s" % request.extension
     747      if request.args:
     748          result += " %s" % request.args
     749      if e['QUERY_STRING']:
     750          result += " ?%s" % e['QUERY_STRING']
     751      if request.uri_language:
     752          result += " (%s)" % request.uri_language
     753      if env:
     754          return request.env
     755      return result
     756
     757
     758 -def filter_err(status, application='app', ticket='tkt'):
     759      "doctest/unittest interface to routes_onerror"
     760      if status > 399 and thread.routes.routes_onerror:
     761          keys = set(('%s/%s' % (application, status),
     762                    '%s/*' % (application),
     763                    '*/%s' % (status),
     764                    '*/*'))
     765          for (key,redir) in thread.routes.routes_onerror:
     766              if key in keys:
     767                  if redir == '!':
     768                      break
     769                  elif '?' in redir:
     770                      url = redir + '&' + 'code=%s&ticket=%s' % (status,ticket)
     771                  else:
     772                      url = redir + '?' + 'code=%s&ticket=%s' % (status,ticket)
     773                  return url # redirection
     774      return status # no action
     775
     776  #  router support
     777  #
     778 -class MapUrlIn(object):
     779      "logic for mapping incoming URLs"
     780
     781 -    def __init__(self, request=None, env=None):
     782          "initialize a map-in object"
     783          self.request = request
     784          self.env = env
     785
     786          self.router = None
     787          self.application = None
     788          self.language = None
     789          self.controller = None
     790          self.function = None
     791          self.extension = 'html'
     792
     793          self.controllers = set()
     794          self.functions = dict()
     795          self.languages = set()
     796          self.default_language = None
     797          self.map_hyphen = False
     798          self.exclusive_domain = False
     799
     800          path = self.env['PATH_INFO']
     801          self.query = self.env.get('QUERY_STRING', None)
     802          path = path.lstrip('/')
     803          self.env['PATH_INFO'] = '/' + path
     804          self.env['WEB2PY_ORIGINAL_URI'] = self.env['PATH_INFO'] + (self.query and ('?' + self.query) or '')
     805
     806          # to handle empty args, strip exactly one trailing slash, if present
     807          # .../arg1// represents one trailing empty arg
     808          #
     809          if path.endswith('/'):
     810              path = path[:-1]
     811          self.args = List(path and path.split('/') or [])
     812
     813          # see http://www.python.org/dev/peps/pep-3333/#url-reconstruction for URL composition
     814          self.remote_addr = self.env.get('REMOTE_ADDR','localhost')
     815          self.scheme = self.env.get('wsgi.url_scheme', 'http').lower()
     816          self.method = self.env.get('REQUEST_METHOD', 'get').lower()
     817          self.host = self.env.get('HTTP_HOST')
     818          self.port = None
     819          if not self.host:
     820              self.host = self.env.get('SERVER_NAME')
     821              self.port = self.env.get('SERVER_PORT')
     822          if not self.host:
     823              self.host = 'localhost'
     824              self.port = '80'
     825          if ':' in self.host:
     826              (self.host, self.port) = self.host.split(':')
     827          if not self.port:
     828              if self.scheme == 'https':
     829                  self.port = '443'
     830              else:
     831                  self.port = '80'
     832
     833 -    def map_prefix(self):
     834          "strip path prefix, if present in its entirety"
     835          prefix = routers.BASE.path_prefix
     836          if prefix:
     837              prefixlen = len(prefix)
     838              if prefixlen > len(self.args):
     839                  return
     840              for i in xrange(prefixlen):
     841                  if prefix[i] != self.args[i]:
     842                      return  # prefix didn't match
     843              self.args = List(self.args[prefixlen:]) # strip the prefix
     844
     845 -    def map_app(self):
     846          "determine application name"
     847          base = routers.BASE  # base router
     848          self.domain_application = None
     849          self.domain_controller = None
     850          self.domain_function = None
     851          arg0 = self.harg0
     852          if (self.host, self.port) in base.domains:
     853              (self.application, self.domain_controller, self.domain_function) = base.domains[(self.host, self.port)]
     854              self.env['domain_application'] = self.application
     855              self.env['domain_controller'] = self.domain_controller
     856              self.env['domain_function'] = self.domain_function
     857          elif (self.host, None) in base.domains:
     858              (self.application, self.domain_controller, self.domain_function) = base.domains[(self.host, None)]
     859              self.env['domain_application'] = self.application
     860              self.env['domain_controller'] = self.domain_controller
     861              self.env['domain_function'] = self.domain_function
     862          elif base.applications and arg0 in base.applications:
     863              self.application = arg0
     864          elif arg0 and not base.applications:
     865              self.application = arg0
     866          else:
     867              self.application = base.default_application or ''
     868          self.pop_arg_if(self.application == arg0)
     869
     870          if not base._acfe_match.match(self.application):
     871              raise HTTP(400, thread.routes.error_message % 'invalid request',
     872                         web2py_error="invalid application: '%s'" % self.application)
     873
     874          if self.application not in routers and \
     875            (self.application != thread.routes.default_application or self.application == 'welcome'):
     876              raise HTTP(400, thread.routes.error_message % 'invalid request',
     877                  web2py_error="unknown application: '%s'" % self.application)
     878
     879          #  set the application router
     880          #
     881          log_rewrite("select application=%s" % self.application)
     882          self.request.application = self.application
     883          if self.application not in routers:
     884              self.router = routers.BASE                # support gluon.main.wsgibase init->welcome
     885          else:
     886              self.router = routers[self.application]   # application router
     887          self.controllers = self.router.controllers
     888          self.default_controller = self.domain_controller or self.router.default_controller
     889          self.functions = self.router.functions
     890          self.languages = self.router.languages
     891          self.default_language = self.router.default_language
     892          self.map_hyphen = self.router.map_hyphen
     893          self.exclusive_domain = self.router.exclusive_domain
     894          self._acfe_match = self.router._acfe_match
     895          self._file_match = self.router._file_match
     896          self._args_match = self.router._args_match
     897
     898 -    def map_root_static(self):
     899          '''
     900          handle root-static files (no hyphen mapping)
     901
     902          a root-static file is one whose incoming URL expects it to be at the root,
     903          typically robots.txt & favicon.ico
     904          '''
     905          if len(self.args) == 1 and self.arg0 in self.router.root_static:
     906              self.controller = self.request.controller = 'static'
     907              root_static_file = os.path.join(self.request.env.applications_parent,
     908                                     'applications', self.application,
     909                                     self.controller, self.arg0)
     910              log_rewrite("route: root static=%s" % root_static_file)
     911              return root_static_file
     912          return None
     913
     914 -    def map_language(self):
     915          "handle language (no hyphen mapping)"
     916          arg0 = self.arg0  # no hyphen mapping
     917          if arg0 and self.languages and arg0 in self.languages:
     918              self.language = arg0
     919          else:
     920              self.language = self.default_language
     921          if self.language:
     922              log_rewrite("route: language=%s" % self.language)
     923              self.pop_arg_if(self.language == arg0)
     924              arg0 = self.arg0
     925
     926 -    def map_controller(self):
     927          "identify controller"
     928          #  handle controller
     929          #
     930          arg0 = self.harg0    # map hyphens
     931          if not arg0 or (self.controllers and arg0 not in self.controllers):
     932              self.controller = self.default_controller or ''
     933          else:
     934              self.controller = arg0
     935          self.pop_arg_if(arg0 == self.controller)
     936          log_rewrite("route: controller=%s" % self.controller)
     937          if not self.router._acfe_match.match(self.controller):
     938              raise HTTP(400, thread.routes.error_message % 'invalid request',
     939                         web2py_error='invalid controller')
     940
     941 -    def map_static(self):
     942          '''
     943          handle static files
     944          file_match but no hyphen mapping
     945          '''
     946          if self.controller != 'static':
     947              return None
     948          file = '/'.join(self.args)
     949          if not self.router._file_match.match(file):
     950              raise HTTP(400, thread.routes.error_message % 'invalid request',
     951                         web2py_error='invalid static file')
     952          #
     953          #  support language-specific static subdirectories,
     954          #  eg /appname/en/static/filename => applications/appname/static/en/filename
     955          #  if language-specific file doesn't exist, try same file in static
     956          #
     957          if self.language:
     958              static_file = os.path.join(self.request.env.applications_parent,
     959                                     'applications', self.application,
     960                                     'static', self.language, file)
     961          if not self.language or not os.path.isfile(static_file):
     962              static_file = os.path.join(self.request.env.applications_parent,
     963                                     'applications', self.application,
     964                                     'static', file)
     965          log_rewrite("route: static=%s" % static_file)
     966          return static_file
     967
     968 -    def map_function(self):
     969          "handle function.extension"
     970          arg0 = self.harg0    # map hyphens
     971          functions = self.functions.get(self.controller, set())
     972          if isinstance(self.router.default_function, dict):
     973              default_function = self.router.default_function.get(self.controller, None)
     974          else:
     975              default_function = self.router.default_function # str or None
     976          default_function = self.domain_function or default_function
     977          if not arg0 or functions and arg0 not in functions:
     978              self.function = default_function or ""
     979              self.pop_arg_if(arg0 and self.function == arg0)
     980          else:
     981              func_ext = arg0.split('.')
     982              if len(func_ext) > 1:
     983                  self.function = func_ext[0]
     984                  self.extension = func_ext[-1]
     985              else:
     986                  self.function = arg0
     987              self.pop_arg_if(True)
     988          log_rewrite("route: function.ext=%s.%s" % (self.function, self.extension))
     989
     990          if not self.router._acfe_match.match(self.function):
     991              raise HTTP(400, thread.routes.error_message % 'invalid request',
     992                         web2py_error='invalid function')
     993          if self.extension and not self.router._acfe_match.match(self.extension):
     994              raise HTTP(400, thread.routes.error_message % 'invalid request',
     995                         web2py_error='invalid extension')
     996
     997 -    def validate_args(self):
     998          '''
     999          check args against validation pattern
    1000          '''
    1001          for arg in self.args:
    1002              if not self.router._args_match.match(arg):
    1003                  raise HTTP(400, thread.routes.error_message % 'invalid request',
    1004                             web2py_error='invalid arg <%s>' % arg)
    1005
    1006 -    def update_request(self):
    1007          '''
    1008          update request from self
    1009          build env.request_uri
    1010          make lower-case versions of http headers in env
    1011          '''
    1012          self.request.application = self.application
    1013          self.request.controller = self.controller
    1014          self.request.function = self.function
    1015          self.request.extension = self.extension
    1016          self.request.args = self.args
    1017          if self.language:
    1018              self.request.uri_language = self.language
    1019          uri = '/%s/%s/%s' % (self.application, self.controller, self.function)
    1020          if self.map_hyphen:
    1021              uri = uri.replace('_', '-')
    1022          if self.extension != 'html':
    1023              uri += '.' + self.extension
    1024          if self.language:
    1025              uri = '/%s%s' % (self.language, uri)
    1026          uri += self.args and urllib.quote('/' + '/'.join([str(x) for x in self.args])) or ''
    1027          uri += (self.query and ('?' + self.query) or '')
    1028          self.env['REQUEST_URI'] = uri
    1029          for (key, value) in self.env.items():
    1030              self.request.env[key.lower().replace('.', '_')] = value
    1031
    1032      @property
    1033 -    def arg0(self):
    1034          "return first arg"
    1035          return self.args(0)
    1036
    1037      @property
    1038 -    def harg0(self):
    1039          "return first arg with optional hyphen mapping"
    1040          if self.map_hyphen and self.args(0):
    1041              return self.args(0).replace('-', '_')
    1042          return self.args(0)
    1043
    1044 -    def pop_arg_if(self, dopop):
    1045          "conditionally remove first arg and return new first arg"
    1046          if dopop:
    1047              self.args.pop(0)
    1048
    1049 -class MapUrlOut(object):
    1050      "logic for mapping outgoing URLs"
    1051
    1052 -    def __init__(self, request, env, application, controller, function, args, other, scheme, host, port):
    1053          "initialize a map-out object"
    1054          self.default_application = routers.BASE.default_application
    1055          if application in routers:
    1056              self.router = routers[application]
    1057          else:
    1058              self.router = routers.BASE
    1059          self.request = request
    1060          self.env = env
    1061          self.application = application
    1062          self.controller = controller
    1063          self.function = function
    1064          self.args = args
    1065          self.other = other
    1066          self.scheme = scheme
    1067          self.host = host
    1068          self.port = port
    1069
    1070          self.applications = routers.BASE.applications
    1071          self.controllers = self.router.controllers
    1072          self.functions = self.router.functions.get(self.controller, set())
    1073          self.languages = self.router.languages
    1074          self.default_language = self.router.default_language
    1075          self.exclusive_domain = self.router.exclusive_domain
    1076          self.map_hyphen = self.router.map_hyphen
    1077          self.map_static = self.router.map_static
    1078          self.path_prefix = routers.BASE.path_prefix
    1079
    1080          self.domain_application = request and self.request.env.domain_application
    1081          self.domain_controller = request and self.request.env.domain_controller
    1082          if isinstance(self.router.default_function, dict):
    1083              self.default_function = self.router.default_function.get(self.controller, None)
    1084          else:
    1085              self.default_function = self.router.default_function
    1086
    1087          if (self.router.exclusive_domain and self.domain_application and self.domain_application != self.application and not self.host):
    1088              raise SyntaxError, 'cross-domain conflict: must specify host'
    1089
    1090          lang = request and request.uri_language
    1091          if lang and self.languages and lang in self.languages:
    1092              self.language = lang
    1093          else:
    1094              self.language = None
    1095
    1096          self.omit_application = False
    1097          self.omit_language = False
    1098          self.omit_controller = False
    1099          self.omit_function = False
    1100
    1101 -    def omit_lang(self):
    1102          "omit language if possible"
    1103
    1104          if not self.language or self.language == self.default_language:
    1105              self.omit_language = True
    1106
    1107 -    def omit_acf(self):
    1108          "omit what we can of a/c/f"
    1109
    1110          router = self.router
    1111
    1112          #  Handle the easy no-args case of tail-defaults: /a/c  /a  /
    1113          #
    1114          if not self.args and self.function == self.default_function:
    1115              self.omit_function = True
    1116              if self.controller == router.default_controller:
    1117                  self.omit_controller = True
    1118                  if self.application == self.default_application:
    1119                      self.omit_application = True
    1120
    1121          #  omit default application
    1122          #  (which might be the domain default application)
    1123          #
    1124          default_application = self.domain_application or self.default_application
    1125          if self.application == default_application:
    1126              self.omit_application = True
    1127
    1128          #  omit controller if default controller
    1129          #
    1130          default_controller = ((self.application == self.domain_application) and self.domain_controller) or router.default_controller or ''
    1131          if self.controller == default_controller:
    1132              self.omit_controller = True
    1133
    1134          #  omit function if possible
    1135          #
    1136          if self.functions and self.function in self.functions and self.function == self.default_function:
    1137              self.omit_function = True
    1138
    1139          #  prohibit ambiguous cases
    1140          #
    1141          #  because we presume the lang string to be unambiguous, its presence protects application omission
    1142          #
    1143          if self.exclusive_domain:
    1144              applications = [self.domain_application]
    1145          else:
    1146              applications = self.applications
    1147          if self.omit_language:
    1148              if not applications or self.controller in applications:
    1149                  self.omit_application = False
    1150              if self.omit_application:
    1151                  if not applications or self.function in applications:
    1152                      self.omit_controller = False
    1153          if not self.controllers or self.function in self.controllers:
    1154              self.omit_controller = False
    1155          if self.args:
    1156              if self.args[0] in self.functions or self.args[0] in self.controllers or self.args[0] in applications:
    1157                  self.omit_function = False
    1158          if self.omit_controller:
    1159              if self.function in self.controllers or self.function in applications:
    1160                  self.omit_controller = False
    1161          if self.omit_application:
    1162              if self.controller in applications:
    1163                  self.omit_application = False
    1164
    1165          #  handle static as a special case
    1166          #  (easier for external static handling)
    1167          #
    1168          if self.controller == 'static' or self.controller.startswith('static/'):
    1169              if not self.map_static:
    1170                  self.omit_application = False
    1171                  if self.language:
    1172                      self.omit_language = False
    1173              self.omit_controller = False
    1174              self.omit_function = False
    1175
    1176 -    def build_acf(self):
    1177          "build acf from components"
    1178          acf = ''
    1179          if self.map_hyphen:
    1180              self.application = self.application.replace('_', '-')
    1181              self.controller = self.controller.replace('_', '-')
    1182              if self.controller != 'static' and not self.controller.startswith('static/'):
    1183                  self.function = self.function.replace('_', '-')
    1184          if not self.omit_application:
    1185              acf += '/' + self.application
    1186          if not self.omit_language:
    1187              acf += '/' + self.language
    1188          if not self.omit_controller:
    1189              acf += '/' + self.controller
    1190          if not self.omit_function:
    1191              acf += '/' + self.function
    1192          if self.path_prefix:
    1193              acf = '/' + '/'.join(self.path_prefix) + acf
    1194          if self.args:
    1195              return acf
    1196          return acf or '/'
    1197
    1198 -    def acf(self):
    1199          "convert components to /app/lang/controller/function"
    1200
    1201          if not routers:
    1202              return None         # use regex filter
    1203          self.omit_lang()        # try to omit language
    1204          self.omit_acf()         # try to omit a/c/f
    1205          return self.build_acf() # build and return the /a/lang/c/f string
    1206
    1207
    1208 -def map_url_in(request, env, app=False):
    1209      "route incoming URL"
    1210
    1211      #  initialize router-url object
    1212      #
    1213      thread.routes = params  # default to base routes
    1214      map = MapUrlIn(request=request, env=env)
    1215      map.map_prefix()  # strip prefix if present
    1216      map.map_app()     # determine application
    1217
    1218      #  configure thread.routes for error rewrite
    1219      #
    1220      if params.routes_app:
    1221          thread.routes = params_apps.get(app, params)
    1222
    1223      if app:
    1224          return map.application
    1225
    1226      root_static_file = map.map_root_static() # handle root-static files
    1227      if root_static_file:
    1228          return (root_static_file, map.env)
    1229      map.map_language()
    1230      map.map_controller()
    1231      static_file = map.map_static()
    1232      if static_file:
    1233          return (static_file, map.env)
    1234      map.map_function()
    1235      map.validate_args()
    1236      map.update_request()
    1237      return (None, map.env)
    1238
    1239 -def map_url_out(request, env, application, controller, function, args, other, scheme, host, port):
    1240      '''
    1241      supply /a/c/f (or /a/lang/c/f) portion of outgoing url
    1242
    1243      The basic rule is that we can only make transformations
    1244      that map_url_in can reverse.
    1245
    1246      Suppose that the incoming arguments are a,c,f,args,lang
    1247      and that the router defaults are da, dc, df, dl.
    1248
    1249      We can perform these transformations trivially if args=[] and lang=None or dl:
    1250
    1251      /da/dc/df => /
    1252      /a/dc/df => /a
    1253      /a/c/df => /a/c
    1254
    1255      We would also like to be able to strip the default application or application/controller
    1256      from URLs with function/args present, thus:
    1257
    1258          /da/c/f/args  => /c/f/args
    1259          /da/dc/f/args => /f/args
    1260
    1261      We use [applications] and [controllers] and {functions} to suppress ambiguous omissions.
    1262
    1263      We assume that language names do not collide with a/c/f names.
    1264      '''
    1265      map = MapUrlOut(request, env, application, controller, function, args, other, scheme, host, port)
    1266      return map.acf()
    1267
    1268 -def get_effective_router(appname):
    1269      "return a private copy of the effective router for the specified application"
    1270      if not routers or appname not in routers:
    1271          return None
    1272      return Storage(routers[appname])  # return a copy
    1273