web2py.gluon.cache

Package web2py :: Package gluon :: Module cache |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.cache

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8
      9  Basic caching classes and methods
     10  =================================
     11
     12  - Cache - The generic caching object interfacing with the others
     13  - CacheInRam - providing caching in ram
     14  - CacheInDisk - provides caches on disk
     15
     16  Memcache is also available via a different module (see gluon.contrib.memcache)
     17
     18  When web2py is running on Google App Engine,
     19  caching will be provided by the GAE memcache
     20  (see gluon.contrib.gae_memcache)
     21  """
     22
     23  import time
     24  import portalocker
     25  import shelve
     26  import thread
     27  import os
     28  import logging
     29  import re
     30
     31  logger = logging.getLogger("web2py.cache")
     32
     33  __all__ = ['Cache']
     34
     35
     36  DEFAULT_TIME_EXPIRE = 300
     37
     38
     39 -class CacheAbstract(object):
     40      """
     41      Abstract class for cache implementations.
     42      Main function is now to provide referenced api documentation.
     43
     44      Use CacheInRam or CacheOnDisk instead which are derived from this class.
     45      """
     46
     47      cache_stats_name = 'web2py_cache_statistics'
     48
     49 -    def __init__(self, request=None):
     50          """
     51          Paremeters
     52          ----------
     53          request:
     54              the global request object
     55          """
     56          raise NotImplementedError
     57
     58 -    def __call__(self, key, f,
     59                  time_expire = DEFAULT_TIME_EXPIRE):
     60          """
     61          Tries retrieve the value corresponding to `key` from the cache of the
     62          object exists and if it did not expire, else it called the function `f`
     63          and stores the output in the cache corresponding to `key`. In the case
     64          the output of the function is returned.
     65
     66          :param key: the key of the object to be store or retrieved
     67          :param f: the function, whose output is to be cached
     68          :param time_expire: expiration of the cache in microseconds
     69
     70          - `time_expire` is used to compare the current time with the time when
     71              the requested object was last saved in cache. It does not affect
     72              future requests.
     73          - Setting `time_expire` to 0 or negative value forces the cache to
     74              refresh.
     75
     76          If the function `f` is `None` the cache is cleared.
     77          """
     78          raise NotImplementedError
     79
     80 -    def clear(self, regex=None):
     81          """
     82          Clears the cache of all keys that match the provided regular expression.
     83          If no regular expression is provided, it clears all entries in cache.
     84
     85          Parameters
     86          ----------
     87          regex:
     88              if provided, only keys matching the regex will be cleared.
     89              Otherwise all keys are cleared.
     90          """
     91
     92          raise NotImplementedError
     93
     94 -    def increment(self, key, value=1):
     95          """
     96          Increments the cached value for the given key by the amount in value
     97
     98          Parameters
     99          ----------
    100          key:
    101              key for the cached object to be incremeneted
    102          value:
    103              amount of the increment (defaults to 1, can be negative)
    104          """
    105          raise NotImplementedError
    106
    107 -    def _clear(self, storage, regex):
    108          """
    109          Auxiliary function called by `clear` to search and clear cache entries
    110          """
    111          r = re.compile(regex)
    112          for (key, value) in storage.items():
    113              if r.match(str(key)):
    114                  del storage[key]
    115
    116 -class CacheInRam(CacheAbstract):
    117      """
    118      Ram based caching
    119
    120      This is implemented as global (per process, shared by all threads)
    121      dictionary.
    122      A mutex-lock mechanism avoid conflicts.
    123      """
    124
    125      locker = thread.allocate_lock()
    126      meta_storage = {}
    127
    128 -    def __init__(self, request=None):
    129          self.locker.acquire()
    130          self.request = request
    131          if request:
    132              app = request.application
    133          else:
    134              app = ''
    135          if not app in self.meta_storage:
    136              self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: {
    137                  'hit_total': 0,
    138                  'misses': 0,
    139              }}
    140          else:
    141              self.storage = self.meta_storage[app]
    142          self.locker.release()
    143
    144 -    def clear(self, regex=None):
    145          self.locker.acquire()
    146          storage = self.storage
    147          if regex is None:
    148              storage.clear()
    149          else:
    150              self._clear(storage, regex)
    151
    152          if not CacheAbstract.cache_stats_name in storage.keys():
    153              storage[CacheAbstract.cache_stats_name] = {
    154                  'hit_total': 0,
    155                  'misses': 0,
    156              }
    157
    158          self.locker.release()
    159
    160 -    def __call__(self, key, f,
    161                  time_expire = DEFAULT_TIME_EXPIRE):
    162          """
    163          Attention! cache.ram does not copy the cached object. It just stores a reference to it.
    164          Turns out the deepcopying the object has some problems:
    165          1) would break backward compatibility
    166          2) would be limiting because people may want to cache live objects
    167          3) would work unless we deepcopy no storage and retrival which would make things slow.
    168          Anyway. You can deepcopy explicitly in the function generating the value to be cached.
    169          """
    170
    171          dt = time_expire
    172
    173          self.locker.acquire()
    174          item = self.storage.get(key, None)
    175          if item and f is None:
    176              del self.storage[key]
    177          self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1
    178          self.locker.release()
    179
    180          if f is None:
    181              return None
    182          if item and (dt is None or item[0] > time.time() - dt):
    183              return item[1]
    184          value = f()
    185
    186          self.locker.acquire()
    187          self.storage[key] = (time.time(), value)
    188          self.storage[CacheAbstract.cache_stats_name]['misses'] += 1
    189          self.locker.release()
    190          return value
    191
    192 -    def increment(self, key, value=1):
    193          self.locker.acquire()
    194          try:
    195              if key in self.storage:
    196                  value = self.storage[key][1] + value
    197              self.storage[key] = (time.time(), value)
    198          except BaseException, e:
    199              self.locker.release()
    200              raise e
    201          self.locker.release()
    202          return value
    203
    204
    205 -class CacheOnDisk(CacheAbstract):
    206      """
    207      Disk based cache
    208
    209      This is implemented as a shelve object and it is shared by multiple web2py
    210      processes (and threads) as long as they share the same filesystem.
    211      The file is locked wen accessed.
    212
    213      Disk cache provides persistance when web2py is started/stopped but it slower
    214      than `CacheInRam`
    215
    216      Values stored in disk cache must be pickable.
    217      """
    218
    219      speedup_checks = set()
    220
    221 -    def __init__(self, request, folder=None):
    222          self.request = request
    223
    224          # Lets test if the cache folder exists, if not
    225          # we are going to create it
    226          folder = folder or os.path.join(request.folder, 'cache')
    227
    228          if not os.path.exists(folder):
    229              os.mkdir(folder)
    230
    231          ### we need this because of a possible bug in shelve that may
    232          ### or may not lock
    233          self.locker_name = os.path.join(folder,'cache.lock')
    234          self.shelve_name = os.path.join(folder,'cache.shelve')
    235
    236          locker, locker_locked = None, False
    237          speedup_key = (folder,CacheAbstract.cache_stats_name)
    238          if not speedup_key in self.speedup_checks or \
    239                  not os.path.exists(self.shelve_name):
    240              try:
    241                  locker = open(self.locker_name, 'a')
    242                  portalocker.lock(locker, portalocker.LOCK_EX)
    243                  locker_locked = True
    244                  storage = shelve.open(self.shelve_name)
    245                  try:
    246                      if not storage.has_key(CacheAbstract.cache_stats_name):
    247                          storage[CacheAbstract.cache_stats_name] = {
    248                              'hit_total': 0,
    249                              'misses': 0,
    250                              }
    251                          storage.sync()
    252                  finally:
    253                      storage.close()
    254                  self.speedup_checks.add(speedup_key)
    255              except ImportError:
    256                  pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
    257              except:
    258                  logger.error('corrupted file %s, will try delete it!' \
    259                                   % self.shelve_name)
    260                  try:
    261                      os.unlink(self.shelve_name)
    262                  except IOError:
    263                      logger.warn('unable to delete file %s' % self.shelve_name)
    264                  except OSError:
    265                      logger.warn('unable to delete file %s' % self.shelve_name)
    266              if locker_locked:
    267                  portalocker.unlock(locker)
    268              if locker:
    269                  locker.close()
    270
    271 -    def clear(self, regex=None):
    272          locker = open(self.locker_name,'a')
    273          portalocker.lock(locker, portalocker.LOCK_EX)
    274          storage = shelve.open(self.shelve_name)
    275          try:
    276              if regex is None:
    277                  storage.clear()
    278              else:
    279                  self._clear(storage, regex)
    280              if not CacheAbstract.cache_stats_name in storage.keys():
    281                  storage[CacheAbstract.cache_stats_name] = {
    282                      'hit_total': 0,
    283                      'misses': 0,
    284                  }
    285              storage.sync()
    286          finally:
    287              storage.close()
    288          portalocker.unlock(locker)
    289          locker.close()
    290
    291 -    def __call__(self, key, f,
    292                  time_expire = DEFAULT_TIME_EXPIRE):
    293          dt = time_expire
    294
    295          locker = open(self.locker_name,'a')
    296          portalocker.lock(locker, portalocker.LOCK_EX)
    297          storage = shelve.open(self.shelve_name)
    298
    299          item = storage.get(key, None)
    300          if item and f is None:
    301              del storage[key]
    302
    303          storage[CacheAbstract.cache_stats_name] = {
    304              'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
    305              'misses': storage[CacheAbstract.cache_stats_name]['misses']
    306          }
    307
    308          storage.sync()
    309
    310          portalocker.unlock(locker)
    311          locker.close()
    312
    313          if f is None:
    314              return None
    315          if item and (dt is None or item[0] > time.time() - dt):
    316              return item[1]
    317          value = f()
    318
    319          locker = open(self.locker_name,'a')
    320          portalocker.lock(locker, portalocker.LOCK_EX)
    321          storage[key] = (time.time(), value)
    322
    323          storage[CacheAbstract.cache_stats_name] = {
    324              'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
    325              'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
    326          }
    327
    328          storage.sync()
    329
    330          storage.close()
    331          portalocker.unlock(locker)
    332          locker.close()
    333
    334          return value
    335
    336 -    def increment(self, key, value=1):
    337          locker = open(self.locker_name,'a')
    338          portalocker.lock(locker, portalocker.LOCK_EX)
    339          storage = shelve.open(self.shelve_name)
    340          try:
    341              if key in storage:
    342                  value = storage[key][1] + value
    343              storage[key] = (time.time(), value)
    344              storage.sync()
    345          finally:
    346              storage.close()
    347              portalocker.unlock(locker)
    348              locker.close()
    349          return value
    350
    351
    352 -class Cache(object):
    353      """
    354      Sets up generic caching, creating an instance of both CacheInRam and
    355      CacheOnDisk.
    356      In case of GAE will make use of gluon.contrib.gae_memcache.
    357
    358      - self.ram is an instance of CacheInRam
    359      - self.disk is an instance of CacheOnDisk
    360      """
    361
    362 -    def __init__(self, request):
    363          """
    364          Parameters
    365          ----------
    366          request:
    367              the global request object
    368          """
    369          # GAE will have a special caching
    370          import settings
    371          if settings.global_settings.web2py_runtime_gae:
    372              from contrib.gae_memcache import MemcacheClient
    373              self.ram=self.disk=MemcacheClient(request)
    374          else:
    375              # Otherwise use ram (and try also disk)
    376              self.ram = CacheInRam(request)
    377              try:
    378                  self.disk = CacheOnDisk(request)
    379              except IOError:
    380                  logger.warning('no cache.disk (IOError)')
    381              except AttributeError:
    382                  # normally not expected anymore, as GAE has already
    383                  # been accounted for
    384                  logger.warning('no cache.disk (AttributeError)')
    385
    386 -    def __call__(self,
    387                  key = None,
    388                  time_expire = DEFAULT_TIME_EXPIRE,
    389                  cache_model = None):
    390          """
    391          Decorator function that can be used to cache any function/method.
    392
    393          Example::
    394
    395              @cache('key', 5000, cache.ram)
    396              def f():
    397                  return time.ctime()
    398
    399          When the function f is called, web2py tries to retrieve
    400          the value corresponding to `key` from the cache of the
    401          object exists and if it did not expire, else it calles the function `f`
    402          and stores the output in the cache corresponding to `key`. In the case
    403          the output of the function is returned.
    404
    405          :param key: the key of the object to be store or retrieved
    406          :param time_expire: expiration of the cache in microseconds
    407          :param cache_model: `cache.ram`, `cache.disk`, or other
    408              (like `cache.memcache` if defined). It defaults to `cache.ram`.
    409
    410          Notes
    411          -----
    412          `time_expire` is used to compare the curret time with the time when the
    413          requested object was last saved in cache. It does not affect future
    414          requests.
    415          Setting `time_expire` to 0 or negative value forces the cache to
    416          refresh.
    417
    418          If the function `f` is an action, we suggest using
    419          `request.env.path_info` as key.
    420          """
    421          if not cache_model:
    422              cache_model = self.ram
    423
    424          def tmp(func):
    425              def action():
    426                  return cache_model(key, func, time_expire)
    427              action.__name___ = func.__name__
    428              action.__doc__ = func.__doc__
    429              return action
    430
    431          return tmp
    432