web2py.gluon.newcron

Package web2py :: Package gluon :: Module newcron |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.newcron

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  Created by Attila Csipa <web2py@csipa.in.rs>
      6  Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  """
      8
      9  import sys
     10  import os
     11  import threading
     12  import logging
     13  import time
     14  import sched
     15  import re
     16  import datetime
     17  import platform
     18  import portalocker
     19  import fileutils
     20  import cPickle
     21  from settings import global_settings
     22
     23  logger = logging.getLogger("web2py.cron")
     24  _cron_stopping = False
     25
     26 -def absolute_path_link(path):
     27      """
     28      Return an absolute path for the destination of a symlink
     29
     30      """
     31      if os.path.islink(path):
     32          link = os.readlink(path)
     33          if not os.path.isabs(link):
     34              link = os.path.join(os.path.dirname(path), link)
     35      else:
     36          link = os.path.abspath(path)
     37      return link
     38
     39 -def stopcron():
     40      "graceful shutdown of cron"
     41      global _cron_stopping
     42      _cron_stopping = True
     43
     44 -class extcron(threading.Thread):
     45
     46 -    def __init__(self, applications_parent):
     47          threading.Thread.__init__(self)
     48          self.setDaemon(False)
     49          self.path = applications_parent
     50          crondance(self.path, 'external', startup=True)
     51
     52 -    def run(self):
     53          if not _cron_stopping:
     54              logger.debug('external cron invocation')
     55              crondance(self.path, 'external', startup=False)
     56
     57 -class hardcron(threading.Thread):
     58
     59 -    def __init__(self, applications_parent):
     60          threading.Thread.__init__(self)
     61          self.setDaemon(True)
     62          self.path = applications_parent
     63          crondance(self.path, 'hard', startup=True)
     64
     65 -    def launch(self):
     66          if not _cron_stopping:
     67              logger.debug('hard cron invocation')
     68              crondance(self.path, 'hard', startup = False)
     69
     70 -    def run(self):
     71          s = sched.scheduler(time.time, time.sleep)
     72          logger.info('Hard cron daemon started')
     73          while not _cron_stopping:
     74              now = time.time()
     75              s.enter(60 - now % 60, 1, self.launch, ())
     76              s.run()
     77
     78 -class softcron(threading.Thread):
     79
     80 -    def __init__(self, applications_parent):
     81          threading.Thread.__init__(self)
     82          self.path = applications_parent
     83          crondance(self.path, 'soft', startup=True)
     84
     85 -    def run(self):
     86          if not _cron_stopping:
     87              logger.debug('soft cron invocation')
     88              crondance(self.path, 'soft', startup=False)
     89
     90 -class Token(object):
     91
     92 -    def __init__(self,path):
     93          self.path = os.path.join(path, 'cron.master')
     94          if not os.path.exists(self.path):
     95              fileutils.write_file(self.path, '', 'wb')
     96          self.master = None
     97          self.now = time.time()
     98
     99 -    def acquire(self,startup=False):
    100          """
    101          returns the time when the lock is acquired or
    102          None if cron already running
    103
    104          lock is implemented by writing a pickle (start, stop) in cron.master
    105          start is time when cron job starts and stop is time when cron completed
    106          stop == 0 if job started but did not yet complete
    107          if a cron job started within less than 60 seconds, acquire returns None
    108          if a cron job started before 60 seconds and did not stop,
    109          a warning is issue "Stale cron.master detected"
    110          """
    111          if portalocker.LOCK_EX is None:
    112              logger.warning('WEB2PY CRON: Disabled because no file locking')
    113              return None
    114          self.master = open(self.path,'rb+')
    115          try:
    116              ret = None
    117              portalocker.lock(self.master,portalocker.LOCK_EX)
    118              try:
    119                  (start, stop) =  cPickle.load(self.master)
    120              except:
    121                  (start, stop) = (0, 1)
    122              if startup or self.now - start > 59.99:
    123                  ret = self.now
    124                  if not stop:
    125                      # this happens if previous cron job longer than 1 minute
    126                      logger.warning('WEB2PY CRON: Stale cron.master detected')
    127                  logger.debug('WEB2PY CRON: Acquiring lock')
    128                  self.master.seek(0)
    129                  cPickle.dump((self.now,0),self.master)
    130          finally:
    131              portalocker.unlock(self.master)
    132          if not ret:
    133              # do this so no need to release
    134              self.master.close()
    135          return ret
    136
    137 -    def release(self):
    138          """
    139          this function writes into cron.master the time when cron job
    140          was completed
    141          """
    142          if not self.master.closed:
    143              portalocker.lock(self.master,portalocker.LOCK_EX)
    144              logger.debug('WEB2PY CRON: Releasing cron lock')
    145              self.master.seek(0)
    146              (start, stop) =  cPickle.load(self.master)
    147              if start == self.now: # if this is my lock
    148                  self.master.seek(0)
    149                  cPickle.dump((self.now,time.time()),self.master)
    150              portalocker.unlock(self.master)
    151              self.master.close()
    152
    153
    154 -def rangetolist(s, period='min'):
    155      retval = []
    156      if s.startswith('*'):
    157          if period == 'min':
    158              s = s.replace('*', '0-59', 1)
    159          elif period == 'hr':
    160              s = s.replace('*', '0-23', 1)
    161          elif period == 'dom':
    162              s = s.replace('*', '1-31', 1)
    163          elif period == 'mon':
    164              s = s.replace('*', '1-12', 1)
    165          elif period == 'dow':
    166              s = s.replace('*', '0-6', 1)
    167      m = re.compile(r'(\d+)-(\d+)/(\d+)')
    168      match = m.match(s)
    169      if match:
    170          for i in range(int(match.group(1)), int(match.group(2)) + 1):
    171              if i % int(match.group(3)) == 0:
    172                  retval.append(i)
    173      return retval
    174
    175
    176 -def parsecronline(line):
    177      task = {}
    178      if line.startswith('@reboot'):
    179          line=line.replace('@reboot', '-1 * * * *')
    180      elif line.startswith('@yearly'):
    181          line=line.replace('@yearly', '0 0 1 1 *')
    182      elif line.startswith('@annually'):
    183          line=line.replace('@annually', '0 0 1 1 *')
    184      elif line.startswith('@monthly'):
    185          line=line.replace('@monthly', '0 0 1 * *')
    186      elif line.startswith('@weekly'):
    187          line=line.replace('@weekly', '0 0 * * 0')
    188      elif line.startswith('@daily'):
    189          line=line.replace('@daily', '0 0 * * *')
    190      elif line.startswith('@midnight'):
    191          line=line.replace('@midnight', '0 0 * * *')
    192      elif line.startswith('@hourly'):
    193          line=line.replace('@hourly', '0 * * * *')
    194      params = line.strip().split(None, 6)
    195      if len(params) < 7:
    196          return None
    197      daysofweek={'sun':0,'mon':1,'tue':2,'wed':3,'thu':4,'fri':5,'sat':6}
    198      for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']):
    199          if not s in [None, '*']:
    200              task[id] = []
    201              vals = s.split(',')
    202              for val in vals:
    203                  if val != '-1' and '-' in val and '/' not in val:
    204                      val = '%s/1' % val
    205                  if '/' in val:
    206                      task[id] += rangetolist(val, id)
    207                  elif val.isdigit() or val=='-1':
    208                      task[id].append(int(val))
    209                  elif id=='dow' and val[:3].lower() in daysofweek:
    210                      task[id].append(daysofweek(val[:3].lower()))
    211      task['user'] = params[5]
    212      task['cmd'] = params[6]
    213      return task
    214
    215
    216 -class cronlauncher(threading.Thread):
    217
    218 -    def __init__(self, cmd, shell=True):
    219          threading.Thread.__init__(self)
    220          if platform.system() == 'Windows':
    221              shell = False
    222          elif isinstance(cmd,list):
    223              cmd = ' '.join(cmd)
    224          self.cmd = cmd
    225          self.shell = shell
    226
    227 -    def run(self):
    228          import subprocess
    229          proc = subprocess.Popen(self.cmd,
    230                                  stdin=subprocess.PIPE,
    231                                  stdout=subprocess.PIPE,
    232                                  stderr=subprocess.PIPE,
    233                                  shell=self.shell)
    234          (stdoutdata,stderrdata) = proc.communicate()
    235          if proc.returncode != 0:
    236              logger.warning(
    237                  'WEB2PY CRON Call returned code %s:\n%s' % \
    238                      (proc.returncode, stdoutdata+stderrdata))
    239          else:
    240              logger.debug('WEB2PY CRON Call returned success:\n%s' \
    241                                % stdoutdata)
    242
    243 -def crondance(applications_parent, ctype='soft', startup=False):
    244      apppath = os.path.join(applications_parent,'applications')
    245      cron_path = os.path.join(applications_parent)
    246      token = Token(cron_path)
    247      cronmaster = token.acquire(startup=startup)
    248      if not cronmaster:
    249          return
    250      now_s = time.localtime()
    251      checks=(('min',now_s.tm_min),
    252              ('hr',now_s.tm_hour),
    253              ('mon',now_s.tm_mon),
    254              ('dom',now_s.tm_mday),
    255              ('dow',(now_s.tm_wday+1)%7))
    256
    257      apps = [x for x in os.listdir(apppath)
    258              if os.path.isdir(os.path.join(apppath, x))]
    259
    260      full_apath_links = set()
    261
    262      for app in apps:
    263          if _cron_stopping:
    264              break;
    265          apath = os.path.join(apppath,app)
    266
    267          # if app is a symbolic link to other app, skip it
    268          full_apath_link = absolute_path_link(apath)
    269          if full_apath_link in full_apath_links:
    270              continue
    271          else:
    272              full_apath_links.add(full_apath_link)
    273
    274          cronpath = os.path.join(apath, 'cron')
    275          crontab = os.path.join(cronpath, 'crontab')
    276          if not os.path.exists(crontab):
    277              continue
    278          try:
    279              cronlines = fileutils.readlines_file(crontab, 'rt')
    280              lines = [x.strip() for x in cronlines if x.strip() and not x.strip().startswith('#')]
    281              tasks = [parsecronline(cline) for cline in lines]
    282          except Exception, e:
    283              logger.error('WEB2PY CRON: crontab read error %s' % e)
    284              continue
    285
    286          for task in tasks:
    287              if _cron_stopping:
    288                  break;
    289              commands = [sys.executable]
    290              w2p_path = fileutils.abspath('web2py.py', gluon=True)
    291              if os.path.exists(w2p_path):
    292                  commands.append(w2p_path)
    293              if global_settings.applications_parent != global_settings.gluon_parent:
    294                  commands.extend(('-f', global_settings.applications_parent))
    295              citems = [(k in task and not v in task[k]) for k,v in checks]
    296              task_min= task.get('min',[])
    297              if not task:
    298                  continue
    299              elif not startup and task_min == [-1]:
    300                  continue
    301              elif task_min != [-1] and reduce(lambda a,b: a or b, citems):
    302                  continue
    303              logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \
    304                               % (ctype, app, task.get('cmd'),
    305                                  os.getcwd(), datetime.datetime.now()))
    306              action, command, models = False, task['cmd'], ''
    307              if command.startswith('**'):
    308                  (action,models,command) = (True,'',command[2:])
    309              elif command.startswith('*'):
    310                  (action,models,command) = (True,'-M',command[1:])
    311              else:
    312                  action=False
    313              if action and command.endswith('.py'):
    314                  commands.extend(('-J',                # cron job
    315                                   models,              # import models?
    316                                   '-S', app,           # app name
    317                                   '-a', '"<recycle>"', # password
    318                                   '-R', command))      # command
    319                  shell = True
    320              elif action:
    321                  commands.extend(('-J',                  # cron job
    322                                   models,                # import models?
    323                                   '-S', app+'/'+command, # app name
    324                                   '-a', '"<recycle>"'))  # password
    325                  shell = True
    326              else:
    327                  commands = command
    328                  shell = False
    329              try:
    330                  cronlauncher(commands, shell=shell).start()
    331              except Exception, e:
    332                  logger.warning(
    333                      'WEB2PY CRON: Execution error for %s: %s' \
    334                          % (task.get('cmd'), e))
    335      token.release()
    336