web2py.gluon.languages

Package web2py :: Package gluon :: Module languages |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.languages

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8  """
      9
     10  import os
     11  import re
     12  import cgi
     13  import portalocker
     14  import logging
     15  import marshal
     16  import copy_reg
     17  from fileutils import listdir
     18  import settings
     19  from cfs import getcfs
     20
     21  __all__ = ['translator', 'findT', 'update_all_languages']
     22
     23  is_gae = settings.global_settings.web2py_runtime_gae
     24
     25  # pattern to find T(blah blah blah) expressions
     26
     27  PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\
     28       + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\
     29       + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\
     30       + r'(?:"(?:[^"\\]|\\.)*"))'
     31
     32  regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL)
     33
     34  # patter for a valid accept_language
     35
     36  regex_language = \
     37      re.compile('^[a-zA-Z]{2}(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$')
     38
     39
     40 -def read_dict_aux(filename):
     41      fp = portalocker.LockedFile(filename, 'r')
     42      lang_text = fp.read().replace('\r\n', '\n')
     43      fp.close()
     44      if not lang_text.strip():
     45          return {}
     46      try:
     47          return eval(lang_text)
     48      except:
     49          logging.error('Syntax error in %s' % filename)
     50          return {}
     51
     52 -def read_dict(filename):
     53      return getcfs('language:%s'%filename,filename,
     54                    lambda filename=filename:read_dict_aux(filename))
     55
     56 -def utf8_repr(s):
     57      r''' # note that we use raw strings to avoid having to use double back slashes below
     58
     59      utf8_repr() works same as repr() when processing ascii string
     60      >>> utf8_repr('abc') == utf8_repr("abc") == repr('abc') == repr("abc") == "'abc'"
     61      True
     62      >>> utf8_repr('a"b"c') == repr('a"b"c') == '\'a"b"c\''
     63      True
     64      >>> utf8_repr("a'b'c") == repr("a'b'c") == '"a\'b\'c"'
     65      True
     66      >>> utf8_repr('a\'b"c') == repr('a\'b"c') == utf8_repr("a'b\"c") == repr("a'b\"c") == '\'a\\\'b"c\''
     67      True
     68      >>> utf8_repr('a\r\nb') == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
     69      True
     70
     71      Unlike repr(), utf8_repr() remains utf8 content when processing utf8 string
     72      >>> utf8_repr('中文字') == utf8_repr("中文字") == "'中文字'" != repr('中文字')
     73      True
     74      >>> utf8_repr('中"文"字') == "'中\"文\"字'" != repr('中"文"字')
     75      True
     76      >>> utf8_repr("中'文'字") == '"中\'文\'字"' != repr("中'文'字")
     77      True
     78      >>> utf8_repr('中\'文"字') == utf8_repr("中'文\"字") == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
     79      True
     80      >>> utf8_repr('中\r\n文') == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
     81      True
     82      '''
     83      if (s.find("'") >= 0) and (s.find('"') < 0): # only single quote exists
     84          s = ''.join(['"', s, '"']) # s = ''.join(['"', s.replace('"','\\"'), '"'])
     85      else:
     86          s = ''.join(["'", s.replace("'","\\'"), "'"])
     87      return s.replace("\n","\\n").replace("\r","\\r")
     88
     89
     90 -def write_dict(filename, contents):
     91      try:
     92          fp = portalocker.LockedFile(filename, 'w')
     93      except (IOError, OSError):
     94          if not is_gae:
     95              logging.warning('Unable to write to file %s' % filename)
     96          return
     97      fp.write('# coding: utf8\n{\n')
     98      for key in sorted(contents):
     99          fp.write('%s: %s,\n' % (utf8_repr(key), utf8_repr(contents[key])))
    100      fp.write('}\n')
    101      fp.close()
    102
    103
    104 -class lazyT(object):
    105
    106      """
    107      never to be called explicitly, returned by translator.__call__
    108      """
    109
    110      m = None
    111      s = None
    112      T = None
    113
    114 -    def __init__(
    115          self,
    116          message,
    117          symbols = {},
    118          T = None,
    119          ):
    120          self.m = message
    121          self.s = symbols
    122          self.T = T
    123
    124 -    def __repr__(self):
    125          return "<lazyT %s>" % (repr(str(self.m)), )
    126
    127 -    def __str__(self):
    128          return self.T.translate(self.m, self.s)
    129
    130 -    def __eq__(self, other):
    131          return self.T.translate(self.m, self.s) == other
    132
    133 -    def __ne__(self, other):
    134          return self.T.translate(self.m, self.s) != other
    135
    136 -    def __add__(self, other):
    137          return '%s%s' % (self, other)
    138
    139 -    def __radd__(self, other):
    140          return '%s%s' % (other, self)
    141
    142 -    def __cmp__(self,other):
    143          return cmp(str(self),str(other))
    144
    145 -    def __hash__(self):
    146          return hash(str(self))
    147
    148 -    def __getattr__(self, name):
    149          return getattr(str(self),name)
    150
    151 -    def __getitem__(self, i):
    152          return str(self)[i]
    153
    154 -    def __getslice__(self, i, j):
    155          return str(self)[i:j]
    156
    157 -    def __iter__(self):
    158          for c in str(self): yield c
    159
    160 -    def __len__(self):
    161          return len(str(self))
    162
    163 -    def xml(self):
    164          return cgi.escape(str(self))
    165
    166 -    def encode(self, *a, **b):
    167          return str(self).encode(*a, **b)
    168
    169 -    def decode(self, *a, **b):
    170          return str(self).decode(*a, **b)
    171
    172 -    def read(self):
    173          return str(self)
    174
    175 -    def __mod__(self, symbols):
    176          return self.T.translate(self.m, symbols)
    177
    178
    179 -class translator(object):
    180
    181      """
    182      this class is instantiated by gluon.compileapp.build_environment
    183      as the T object
    184
    185      ::
    186
    187          T.force(None) # turns off translation
    188          T.force('fr, it') # forces web2py to translate using fr.py or it.py
    189
    190          T(\"Hello World\") # translates \"Hello World\" using the selected file
    191
    192      notice 1: there is no need to force since, by default, T uses
    193      accept_language to determine a translation file.
    194
    195      notice 2: en and en-en are considered different languages!
    196      """
    197
    198 -    def __init__(self, request):
    199          self.request = request
    200          self.folder = request.folder
    201          self.current_languages = ['en']
    202          self.accepted_language = None
    203          self.language_file = None
    204          self.http_accept_language = request.env.http_accept_language
    205          self.requested_languages = self.force(self.http_accept_language)
    206          self.lazy = True
    207          self.otherTs = {}
    208
    209 -    def get_possible_languages(self):
    210          possible_languages = [lang for lang in self.current_languages]
    211          file_ending = re.compile("\.py$")
    212          for langfile in os.listdir(os.path.join(self.folder,'languages')):
    213              if file_ending.search(langfile):
    214                  possible_languages.append(file_ending.sub('',langfile))
    215          return possible_languages
    216
    217 -    def set_current_languages(self, *languages):
    218          if len(languages) == 1 and isinstance(languages[0], (tuple, list)):
    219              languages = languages[0]
    220          self.current_languages = languages
    221          self.force(self.http_accept_language)
    222
    223 -    def force(self, *languages):
    224          if not languages or languages[0] is None:
    225              languages = []
    226          if len(languages) == 1 and isinstance(languages[0], (str, unicode)):
    227              languages = languages[0]
    228          if languages:
    229              if isinstance(languages, (str, unicode)):
    230                  accept_languages = languages.split(';')
    231                  languages = []
    232                  [languages.extend(al.split(',')) for al in accept_languages]
    233                  languages = [item.strip().lower() for item in languages \
    234                                   if regex_language.match(item.strip())]
    235
    236              for language in languages:
    237                  if language in self.current_languages:
    238                      self.accepted_language = language
    239                      break
    240                  filename = os.path.join(self.folder, 'languages/', language + '.py')
    241                  if os.path.exists(filename):
    242                      self.accepted_language = language
    243                      self.language_file = filename
    244                      self.t = read_dict(filename)
    245                      return languages
    246          self.language_file = None
    247          self.t = {}  # ## no language by default
    248          return languages
    249
    250 -    def __call__(self, message, symbols={}, language=None, lazy=None):
    251          if lazy is None:
    252              lazy = self.lazy
    253          if not language:
    254              if lazy:
    255                  return lazyT(message, symbols, self)
    256              else:
    257                  return self.translate(message, symbols)
    258          else:
    259              try:
    260                  otherT = self.otherTs[language]
    261              except KeyError:
    262                  otherT = self.otherTs[language] = translator(self.request)
    263                  otherT.force(language)
    264              return otherT(message, symbols, lazy=lazy)
    265
    266 -    def translate(self, message, symbols):
    267          """
    268          user ## to add a comment into a translation string
    269          the comment can be useful do discriminate different possible
    270          translations for the same string (for example different locations)
    271
    272          T(' hello world ') -> ' hello world '
    273          T(' hello world ## token') -> 'hello world'
    274          T('hello ## world ## token') -> 'hello ## world'
    275
    276          the ## notation is ignored in multiline strings and strings that
    277          start with ##. this is to allow markmin syntax to be translated
    278          """
    279          #for some reason languages.py gets executed before gaehandler.py
    280          # is able to set web2py_runtime_gae, so re-check here
    281          is_gae = settings.global_settings.web2py_runtime_gae
    282          if not message.startswith('#') and not '\n' in message:
    283              tokens = message.rsplit('##', 1)
    284          else:
    285              # this allows markmin syntax in translations
    286              tokens = [message]
    287          if len(tokens) == 2:
    288              tokens[0] = tokens[0].strip()
    289              message = tokens[0] + '##' + tokens[1].strip()
    290          mt = self.t.get(message, None)
    291          if mt is None:
    292              self.t[message] = mt = tokens[0]
    293              if self.language_file and not is_gae:
    294                  write_dict(self.language_file, self.t)
    295          if symbols or symbols == 0:
    296              return mt % symbols
    297          return mt
    298
    299
    300 -def findT(path, language='en-us'):
    301      """
    302      must be run by the admin app
    303      """
    304      filename = os.path.join(path, 'languages', '%s.py' % language)
    305      sentences = read_dict(filename)
    306      mp = os.path.join(path, 'models')
    307      cp = os.path.join(path, 'controllers')
    308      vp = os.path.join(path, 'views')
    309      for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\
    310           + listdir(vp, '.+\.html', 0):
    311          fp = portalocker.LockedFile(file, 'r')
    312          data = fp.read()
    313          fp.close()
    314          items = regex_translate.findall(data)
    315          for item in items:
    316              try:
    317                  message = eval(item)
    318                  if not message.startswith('#') and not '\n' in message:
    319                      tokens = message.rsplit('##', 1)
    320                  else:
    321                      # this allows markmin syntax in translations
    322                      tokens = [message]
    323                  if len(tokens) == 2:
    324                      message = tokens[0].strip() + '##' + tokens[1].strip()
    325                  if message and not message in sentences:
    326                      sentences[message] = message
    327              except:
    328                  pass
    329      write_dict(filename, sentences)
    330
    331  ### important to allow safe session.flash=T(....)
    332 -def lazyT_unpickle(data):
    333      return marshal.loads(data)
    334 -def lazyT_pickle(data):
    335      return lazyT_unpickle, (marshal.dumps(str(data)),)
    336  copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle)
    337
    338 -def update_all_languages(application_path):
    339      path = os.path.join(application_path, 'languages/')
    340      for language in listdir(path, '^\w+(\-\w+)?\.py$'):
    341          findT(application_path, language[:-3])
    342
    343
    344  if __name__ == '__main__':
    345      import doctest
    346      doctest.testmod()
    347